00000080:00200000:2.0:1789762970.358310:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1789762970.358332:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2822 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.358358:0:16279:0:(rw26.c:871:ll_write_end()) pos 11558912, len 4096, copied 4096 00000080:00200000:2.0:1789762970.358406:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003d6cfa93[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2822] 00000080:00200000:2.0:1789762970.358434:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000051be0b85 17ffffc0001001 3:0 ffff936010e038c0 2822 lru 00000080:00200000:2.0:1789762970.358514:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003d3b3607 1798: 1< 0 - - > 2< 7364608 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.358531:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003d6cfa93 00000080:00200000:2.0:1789762970.358539:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1789762970.358545:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2823 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.358570:0:16279:0:(rw26.c:871:ll_write_end()) pos 11563008, len 4096, copied 4096 00000080:00200000:2.0:1789762970.358585:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000cb3be38d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2823] 00000080:00200000:2.0:1789762970.358608:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007b57bb00 17ffffc0001001 3:0 ffff936010e037e0 2823 lru 00000080:00200000:2.0:1789762970.358678:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000997b4524 1799: 1< 0 - - > 2< 7368704 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.358685:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000cb3be38d 00000080:00200000:2.0:1789762970.358688:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1789762970.358696:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2824 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.358719:0:16279:0:(rw26.c:871:ll_write_end()) pos 11567104, len 4096, copied 4096 00000080:00200000:2.0:1789762970.358769:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ae9fb60b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2824] 00000080:00200000:2.0:1789762970.358784:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005a881551 17ffffc0001001 3:0 ffff936010e039a0 2824 lru 00000080:00200000:2.0:1789762970.358811:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000005d0ea2a 1800: 1< 0 - - > 2< 7372800 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.358815:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ae9fb60b 00000080:00200000:2.0:1789762970.358817:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1789762970.358820:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2825 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.358958:0:16279:0:(rw26.c:871:ll_write_end()) pos 11571200, len 4096, copied 4096 00000080:00200000:2.0:1789762970.358976:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005b0bc33b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2825] 00000080:00200000:2.0:1789762970.358983:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000982d3259 17ffffc0001001 3:0 ffff936010e03540 2825 lru 00000080:00200000:2.0:1789762970.359010:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000dd090fc9 1801: 1< 0 - - > 2< 7376896 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.359022:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005b0bc33b 00000080:00200000:2.0:1789762970.359032:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1789762970.359045:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2826 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.359093:0:16279:0:(rw26.c:871:ll_write_end()) pos 11575296, len 4096, copied 4096 00000080:00200000:2.0:1789762970.359133:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a34164d2[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2826] 00000080:00200000:2.0:1789762970.359161:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000079aa4bb9 17ffffc0001001 3:0 ffff936010e03b60 2826 lru 00000080:00200000:2.0:1789762970.359229:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003e317bde 1802: 1< 0 - - > 2< 7380992 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.359239:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a34164d2 00000080:00200000:2.0:1789762970.359253:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1789762970.359267:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2827 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.359293:0:16279:0:(rw26.c:871:ll_write_end()) pos 11579392, len 4096, copied 4096 00000080:00200000:2.0:1789762970.359360:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009dc56451[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2827] 00000080:00200000:2.0:1789762970.359389:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000039cbec43 17ffffc0001001 3:0 ffff936010e030e0 2827 lru 00000080:00200000:2.0:1789762970.359431:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000036a937e3 1803: 1< 0 - - > 2< 7385088 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.359452:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009dc56451 00000080:00200000:2.0:1789762970.359459:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1789762970.359470:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2828 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.359510:0:16279:0:(rw26.c:871:ll_write_end()) pos 11583488, len 4096, copied 4096 00000080:00200000:2.0:1789762970.359547:0:16279:0:(rw26.c:929:ll_write_end()) page@000000007bb8c6ba[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2828] 00000080:00200000:2.0:1789762970.359572:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000030688c8 17ffffc0001001 3:0 ffff936010e03620 2828 lru 00000080:00200000:2.0:1789762970.359624:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004d8ce8e5 1804: 1< 0 - - > 2< 7389184 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.359634:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000007bb8c6ba 00000080:00200000:2.0:1789762970.359641:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1789762970.359651:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2829 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.359692:0:16279:0:(rw26.c:871:ll_write_end()) pos 11587584, len 4096, copied 4096 00000080:00200000:2.0:1789762970.359716:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000033f8660d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2829] 00000080:00200000:2.0:1789762970.359757:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000001abd49e 17ffffc0001001 3:0 ffff936010e03ee0 2829 lru 00000080:00200000:2.0:1789762970.359802:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000052a18570 1805: 1< 0 - - > 2< 7393280 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.359810:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000033f8660d 00000080:00200000:2.0:1789762970.359815:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1789762970.359821:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2830 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.359939:0:16279:0:(rw26.c:871:ll_write_end()) pos 11591680, len 4096, copied 4096 00000080:00200000:2.0:1789762970.359973:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e5953368[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2830] 00000080:00200000:2.0:1789762970.359988:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000eabe1f6e 17ffffc0001021 2:0 ffff936010e03460 2830 lru 00000080:00200000:2.0:1789762970.360014:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000332ccdc 1806: 1< 0 - - > 2< 7397376 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360021:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e5953368 00000080:00200000:2.0:1789762970.360033:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1789762970.360044:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2831 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360094:0:16279:0:(rw26.c:871:ll_write_end()) pos 11595776, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360099:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fa729178[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2831] 00000080:00200000:2.0:1789762970.360101:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e8b7f651 17ffffc0001001 3:0 ffff936010e03380 2831 lru 00000080:00200000:2.0:1789762970.360106:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000080381f9e 1807: 1< 0 - - > 2< 7401472 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360112:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fa729178 00000080:00200000:2.0:1789762970.360122:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1789762970.360127:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2832 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360146:0:16279:0:(rw26.c:871:ll_write_end()) pos 11599872, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360167:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000068cd4055[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2832] 00000080:00200000:2.0:1789762970.360189:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008e6dd418 17ffffc0001001 3:0 ffff936010e03d20 2832 lru 00000080:00200000:2.0:1789762970.360263:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000907f075e 1808: 1< 0 - - > 2< 7405568 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360283:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000068cd4055 00000080:00200000:2.0:1789762970.360297:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1789762970.360313:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2833 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360374:0:16279:0:(rw26.c:871:ll_write_end()) pos 11603968, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360395:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fda04c5f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2833] 00000080:00200000:2.0:1789762970.360434:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000420a92f4 17ffffc0001001 3:0 ffff936010e03000 2833 lru 00000080:00200000:2.0:1789762970.360495:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a8bc6676 1809: 1< 0 - - > 2< 7409664 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360508:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fda04c5f 00000080:00200000:2.0:1789762970.360517:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1789762970.360527:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2834 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360564:0:16279:0:(rw26.c:871:ll_write_end()) pos 11608064, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360590:0:16279:0:(rw26.c:929:ll_write_end()) page@000000000fd9bad4[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2834] 00000080:00200000:2.0:1789762970.360613:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000fd43c537 17ffffc0001001 3:0 ffff936010e032a0 2834 lru 00000080:00200000:2.0:1789762970.360653:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000026282599 1810: 1< 0 - - > 2< 7413760 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360682:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000000fd9bad4 00000080:00200000:2.0:1789762970.360691:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1789762970.360698:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2835 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360724:0:16279:0:(rw26.c:871:ll_write_end()) pos 11612160, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360774:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000088f1db99[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2835] 00000080:00200000:2.0:1789762970.360791:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000db41b1c5 17ffffc0001001 3:0 ffff936010e03e00 2835 lru 00000080:00200000:2.0:1789762970.360804:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000094c8ab4b 1811: 1< 0 - - > 2< 7417856 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.360809:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000088f1db99 00000080:00200000:2.0:1789762970.360812:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1789762970.360815:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2836 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.360845:0:16279:0:(rw26.c:871:ll_write_end()) pos 11616256, len 4096, copied 4096 00000080:00200000:2.0:1789762970.360940:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000011b846e1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2836] 00000080:00200000:2.0:1789762970.360958:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d07f9ae9 17ffffc0001001 3:0 ffff936010e031c0 2836 lru 00000080:00200000:2.0:1789762970.361008:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000075f9c733 1812: 1< 0 - - > 2< 7421952 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361019:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000011b846e1 00000080:00200000:2.0:1789762970.361030:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1789762970.361041:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2837 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361061:0:16279:0:(rw26.c:871:ll_write_end()) pos 11620352, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361090:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000010fc6b31[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2837] 00000080:00200000:2.0:1789762970.361119:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c697e748 17ffffc0001001 3:0 ffff936010e03c40 2837 lru 00000080:00200000:2.0:1789762970.361180:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000039c4862b 1813: 1< 0 - - > 2< 7426048 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361185:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000010fc6b31 00000080:00200000:2.0:1789762970.361187:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1789762970.361191:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2838 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361234:0:16279:0:(rw26.c:871:ll_write_end()) pos 11624448, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361272:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b2d3daf4[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2838] 00000080:00200000:2.0:1789762970.361289:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004098a431 17ffffc0001001 3:0 ffff93601a65b620 2838 lru 00000080:00200000:2.0:1789762970.361361:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b1e54bdc 1814: 1< 0 - - > 2< 7430144 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361371:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b2d3daf4 00000080:00200000:2.0:1789762970.361378:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1789762970.361390:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2839 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361452:0:16279:0:(rw26.c:871:ll_write_end()) pos 11628544, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361468:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000017b30a01[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2839] 00000080:00200000:2.0:1789762970.361486:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d0079c11 17ffffc0001001 3:0 ffff93601a65bee0 2839 lru 00000080:00200000:2.0:1789762970.361517:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e997c35f 1815: 1< 0 - - > 2< 7434240 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361536:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000017b30a01 00000080:00200000:2.0:1789762970.361548:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1789762970.361556:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2840 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361589:0:16279:0:(rw26.c:871:ll_write_end()) pos 11632640, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361623:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000097af3643[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2840] 00000080:00200000:2.0:1789762970.361645:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000571cd670 17ffffc0001001 3:0 ffff93601a65b460 2840 lru 00000080:00200000:2.0:1789762970.361685:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000057b03eb7 1816: 1< 0 - - > 2< 7438336 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361688:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000097af3643 00000080:00200000:2.0:1789762970.361691:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1789762970.361697:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2841 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361716:0:16279:0:(rw26.c:871:ll_write_end()) pos 11636736, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361764:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000de026126[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2841] 00000080:00200000:2.0:1789762970.361794:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009314578a 17ffffc0001001 3:0 ffff93601a65b380 2841 lru 00000080:00200000:2.0:1789762970.361819:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008398d72b 1817: 1< 0 - - > 2< 7442432 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.361830:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000de026126 00000080:00200000:2.0:1789762970.361845:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1789762970.361896:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2842 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.361941:0:16279:0:(rw26.c:871:ll_write_end()) pos 11640832, len 4096, copied 4096 00000080:00200000:2.0:1789762970.361966:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000099185ab5[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2842] 00000080:00200000:2.0:1789762970.362002:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d19d5c33 17ffffc0001001 3:0 ffff93601a65bd20 2842 lru 00000080:00200000:2.0:1789762970.362049:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000751982c3 1818: 1< 0 - - > 2< 7446528 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362061:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000099185ab5 00000080:00200000:2.0:1789762970.362069:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1789762970.362080:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2843 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362113:0:16279:0:(rw26.c:871:ll_write_end()) pos 11644928, len 4096, copied 4096 00000080:00200000:2.0:1789762970.362132:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ca303a7c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2843] 00000080:00200000:2.0:1789762970.362158:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008787bf95 17ffffc0001001 3:0 ffff93601a65b000 2843 lru 00000080:00200000:2.0:1789762970.362217:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b12e958d 1819: 1< 0 - - > 2< 7450624 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362235:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ca303a7c 00000080:00200000:2.0:1789762970.362252:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1789762970.362266:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2844 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362297:0:16279:0:(rw26.c:871:ll_write_end()) pos 11649024, len 4096, copied 4096 00000080:00200000:2.0:1789762970.362314:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002bef1131[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2844] 00000080:00200000:2.0:1789762970.362370:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e6a82834 17ffffc0001001 3:0 ffff93601a65b2a0 2844 lru 00000080:00200000:2.0:1789762970.362447:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000049cbbf54 1820: 1< 0 - - > 2< 7454720 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362462:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002bef1131 00000080:00200000:2.0:1789762970.362472:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1789762970.362479:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2845 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362514:0:16279:0:(rw26.c:871:ll_write_end()) pos 11653120, len 4096, copied 4096 00000080:00200000:2.0:1789762970.362540:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000043dafc87[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2845] 00000080:00200000:2.0:1789762970.362564:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000068bd5ca8 17ffffc0001021 2:0 ffff93601a65be00 2845 lru 00000080:00200000:2.0:1789762970.362585:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a7d7b950 1821: 1< 0 - - > 2< 7458816 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362593:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000043dafc87 00000080:00200000:2.0:1789762970.362603:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1789762970.362613:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2846 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362649:0:16279:0:(rw26.c:871:ll_write_end()) pos 11657216, len 4096, copied 4096 00000080:00200000:2.0:1789762970.362689:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005e31f836[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2846] 00000080:00200000:2.0:1789762970.362704:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e80798db 17ffffc0001001 3:0 ffff93601a65b1c0 2846 lru 00000080:00200000:2.0:1789762970.362737:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a6d9115e 1822: 1< 0 - - > 2< 7462912 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362747:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005e31f836 00000080:00200000:2.0:1789762970.362760:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1789762970.362772:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2847 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362805:0:16279:0:(rw26.c:871:ll_write_end()) pos 11661312, len 4096, copied 4096 00000080:00200000:2.0:1789762970.362819:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000db5773e4[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2847] 00000080:00200000:2.0:1789762970.362829:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000081643623 17ffffc0001001 3:0 ffff93601a65bc40 2847 lru 00000080:00200000:2.0:1789762970.362931:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bf7ba5ee 1823: 1< 0 - - > 2< 7467008 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.362941:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000db5773e4 00000080:00200000:2.0:1789762970.362946:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1789762970.362952:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2848 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.362974:0:16279:0:(rw26.c:871:ll_write_end()) pos 11665408, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363008:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000070d685f9[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2848] 00000080:00200000:2.0:1789762970.363037:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f6c87b1a 17ffffc0001001 3:0 ffff93601a65ba80 2848 lru 00000080:00200000:2.0:1789762970.363081:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ef5553f5 1824: 1< 0 - - > 2< 7471104 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363093:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000070d685f9 00000080:00200000:2.0:1789762970.363100:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1789762970.363110:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2849 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.363135:0:16279:0:(rw26.c:871:ll_write_end()) pos 11669504, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363166:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ecd03d9b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2849] 00000080:00200000:2.0:1789762970.363187:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005922ba1c 17ffffc0001001 3:0 ffff93601a65b700 2849 lru 00000080:00200000:2.0:1789762970.363262:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003e5b0623 1825: 1< 0 - - > 2< 7475200 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363275:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ecd03d9b 00000080:00200000:2.0:1789762970.363283:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1789762970.363295:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2850 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.363361:0:16279:0:(rw26.c:871:ll_write_end()) pos 11673600, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363373:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009f07000d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2850] 00000080:00200000:2.0:1789762970.363382:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000090fad321 17ffffc0001001 3:0 ffff93601a65b8c0 2850 lru 00000080:00200000:2.0:1789762970.363447:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000044d440e6 1826: 1< 0 - - > 2< 7479296 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363462:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009f07000d 00000080:00200000:2.0:1789762970.363468:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1789762970.363478:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2851 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.363514:0:16279:0:(rw26.c:871:ll_write_end()) pos 11677696, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363525:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e3f6ff9d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2851] 00000080:00200000:2.0:1789762970.363533:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000035d0e5de 17ffffc0001001 3:0 ffff93601a65b7e0 2851 lru 00000080:00200000:2.0:1789762970.363584:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000090522f23 1827: 1< 0 - - > 2< 7483392 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363596:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e3f6ff9d 00000080:00200000:2.0:1789762970.363605:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1789762970.363617:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2852 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.363666:0:16279:0:(rw26.c:871:ll_write_end()) pos 11681792, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363683:0:16279:0:(rw26.c:929:ll_write_end()) page@000000007d07c6b3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2852] 00000080:00200000:2.0:1789762970.363698:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000644fd0fb 17ffffc0001001 3:0 ffff93601a65b9a0 2852 lru 00000080:00200000:2.0:1789762970.363719:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008d92d8f1 1828: 1< 0 - - > 2< 7487488 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363722:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000007d07c6b3 00000080:00200000:2.0:1789762970.363724:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1789762970.363727:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2853 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.363778:0:16279:0:(rw26.c:871:ll_write_end()) pos 11685888, len 4096, copied 4096 00000080:00200000:2.0:1789762970.363803:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000031d35bee[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2853] 00000080:00200000:2.0:1789762970.363817:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000029735e05 17ffffc0001001 3:0 ffff93601a65b540 2853 lru 00000080:00200000:2.0:1789762970.363937:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000018f8fe35 1829: 1< 0 - - > 2< 7491584 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.363949:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000031d35bee 00000080:00200000:2.0:1789762970.363952:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 38. 00010000:00010000:2.0:1789762970.363975:0:16269:0:(ldlm_request.c:193:ldlm_completion_tail()) ### client-side enqueue: granted ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x20000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.364038:0:16269:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x20000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.364099:0:16269:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x10000000020000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1789762970.364155:0:16269:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 1048576, setting rss=1048576; kms=1048576, end=2097151 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 4/0,2 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x10020000020000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.364243:0:16269:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 5/0,2 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x810020000020000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.364312:0:16269:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000023ca0103/0xb758ac0edef9cb42 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [1048576->2097151] (req 1048576->2097151) gid 0 flags: 0x810020000020000 nid: local remote: 0xdf5a7ac59f177a7a expref: -99 pid: 16269 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1789762970.364415:0:16269:0:(vvp_io.c:1292:vvp_io_write_start()) nrs_w_oleg337-client.virtnet: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:2.0:1789762970.364456:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 256 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.364496:0:16269:0:(rw26.c:871:ll_write_end()) pos 1048576, len 4096, copied 4096 00000080:00200000:2.0:1789762970.364523:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000abda0030[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 256] 00000080:00200000:2.0:1789762970.364546:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000078b21d46 17ffffc0001001 3:0 ffff93601a65bb60 256 lru 00000080:00200000:2.0:1789762970.364598:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000e615ed9 256: 1< 0 - - > 2< 1048576 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.364610:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000abda0030 00000080:00200000:2.0:1789762970.364616:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1789762970.364626:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 257 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.364659:0:16269:0:(rw26.c:871:ll_write_end()) pos 1052672, len 4096, copied 4096 00000080:00200000:2.0:1789762970.364696:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f8ae916a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 257] 00000080:00200000:2.0:1789762970.364709:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007fb63649 17ffffc0001001 3:0 ffff93601a65b0e0 257 lru 00000080:00200000:2.0:1789762970.364766:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b54b1968 257: 1< 0 - - > 2< 1052672 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.364775:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f8ae916a 00000080:00200000:2.0:1789762970.364785:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 2. 00000080:00200000:2.0:1789762970.364795:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 258 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.364829:0:16269:0:(rw26.c:871:ll_write_end()) pos 1056768, len 4096, copied 4096 00000080:00200000:2.0:1789762970.364925:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000fce607ba[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 258] 00000080:00200000:2.0:1789762970.364946:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009a9ccb53 17ffffc0001001 3:0 ffff93601a6b07e0 258 lru 00000080:00200000:2.0:1789762970.365018:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000008fe65702 258: 1< 0 - - > 2< 1056768 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365033:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000fce607ba 00000080:00200000:2.0:1789762970.365040:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 3. 00000080:00200000:2.0:1789762970.365052:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 259 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365085:0:16269:0:(rw26.c:871:ll_write_end()) pos 1060864, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365108:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000dd09871e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 259] 00000080:00200000:2.0:1789762970.365138:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b0226462 17ffffc0001001 3:0 ffff93601a6b09a0 259 lru 00000080:00200000:2.0:1789762970.365214:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000419b3f3a 259: 1< 0 - - > 2< 1060864 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365227:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000dd09871e 00000080:00200000:2.0:1789762970.365235:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 4. 00000080:00200000:2.0:1789762970.365242:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 260 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365274:0:16269:0:(rw26.c:871:ll_write_end()) pos 1064960, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365283:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000b5558897[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 260] 00000080:00200000:2.0:1789762970.365287:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000097e55dd4 17ffffc0001001 3:0 ffff93601a6b0540 260 lru 00000080:00200000:2.0:1789762970.365308:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000376189a1 260: 1< 0 - - > 2< 1064960 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365330:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000b5558897 00000080:00200000:2.0:1789762970.365335:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 5. 00000080:00200000:2.0:1789762970.365345:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 261 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365399:0:16269:0:(rw26.c:871:ll_write_end()) pos 1069056, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365439:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ccb9c646[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 261] 00000080:00200000:2.0:1789762970.365456:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000003c3ffc6 17ffffc0001001 3:0 ffff93601a6b0b60 261 lru 00000080:00200000:2.0:1789762970.365485:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000bb14d9b8 261: 1< 0 - - > 2< 1069056 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365493:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ccb9c646 00000080:00200000:2.0:1789762970.365499:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 6. 00000080:00200000:2.0:1789762970.365511:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 262 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365550:0:16269:0:(rw26.c:871:ll_write_end()) pos 1073152, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365580:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004bbb8a13[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 262] 00000080:00200000:2.0:1789762970.365606:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000003c71804f 17ffffc0001021 2:0 ffff93601a6b00e0 262 lru 00000080:00200000:2.0:1789762970.365674:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c5b826bb 262: 1< 0 - - > 2< 1073152 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365678:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004bbb8a13 00000080:00200000:2.0:1789762970.365680:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 7. 00000080:00200000:2.0:1789762970.365685:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 263 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365705:0:16269:0:(rw26.c:871:ll_write_end()) pos 1077248, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365750:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005e26f9ca[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 263] 00000080:00200000:2.0:1789762970.365789:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001d67a0d4 17ffffc0001001 3:0 ffff93601a6b0620 263 lru 00000080:00200000:2.0:1789762970.365808:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000002556f2f2 263: 1< 0 - - > 2< 1077248 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.365815:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005e26f9ca 00000080:00200000:2.0:1789762970.365821:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 8. 00000080:00200000:2.0:1789762970.365835:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 264 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.365908:0:16269:0:(rw26.c:871:ll_write_end()) pos 1081344, len 4096, copied 4096 00000080:00200000:2.0:1789762970.365941:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000004f2d407[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 264] 00000080:00200000:2.0:1789762970.365962:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000017e07c49 17ffffc0001001 3:0 ffff93601a6b0ee0 264 lru 00000080:00200000:2.0:1789762970.366016:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000092bf76e 264: 1< 0 - - > 2< 1081344 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366026:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000004f2d407 00000080:00200000:2.0:1789762970.366036:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 9. 00000080:00200000:2.0:1789762970.366043:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 265 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.366077:0:16269:0:(rw26.c:871:ll_write_end()) pos 1085440, len 4096, copied 4096 00000080:00200000:2.0:1789762970.366109:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000fb35f5ef[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 265] 00000080:00200000:2.0:1789762970.366134:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000017ed36d9 17ffffc0001001 3:0 ffff93601a6b0460 265 lru 00000080:00200000:2.0:1789762970.366203:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f43f2ccf 265: 1< 0 - - > 2< 1085440 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366233:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000fb35f5ef 00000080:00200000:2.0:1789762970.366239:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 10. 00000080:00200000:2.0:1789762970.366251:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 266 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.366282:0:16269:0:(rw26.c:871:ll_write_end()) pos 1089536, len 4096, copied 4096 00000080:00200000:2.0:1789762970.366335:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005d6c1205[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 266] 00000080:00200000:2.0:1789762970.366371:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e395f636 17ffffc0001001 3:0 ffff93601a6b0380 266 lru 00000080:00200000:2.0:1789762970.366433:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005f01086d 266: 1< 0 - - > 2< 1089536 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366446:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005d6c1205 00000080:00200000:2.0:1789762970.366452:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 11. 00000080:00200000:2.0:1789762970.366463:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 267 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.366495:0:16269:0:(rw26.c:871:ll_write_end()) pos 1093632, len 4096, copied 4096 00000080:00200000:2.0:1789762970.366530:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000286d004d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 267] 00000080:00200000:2.0:1789762970.366548:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000198aa57d 17ffffc0001001 3:0 ffff93601a6b0d20 267 lru 00000080:00200000:2.0:1789762970.366598:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e6ae4e9e 267: 1< 0 - - > 2< 1093632 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366611:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000286d004d 00000080:00200000:2.0:1789762970.366621:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 12. 00000080:00200000:2.0:1789762970.366633:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 268 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.366678:0:16269:0:(rw26.c:871:ll_write_end()) pos 1097728, len 4096, copied 4096 00000080:00200000:2.0:1789762970.366696:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d3ac54f2[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 268] 00000080:00200000:2.0:1789762970.366716:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000de1de615 17ffffc0001001 3:0 ffff93601a6b0000 268 lru 00000080:00200000:2.0:1789762970.366759:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000317390eb 268: 1< 0 - - > 2< 1097728 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366772:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d3ac54f2 00000080:00200000:2.0:1789762970.366783:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 13. 00000080:00200000:2.0:1789762970.366793:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 269 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.366820:0:16269:0:(rw26.c:871:ll_write_end()) pos 1101824, len 4096, copied 4096 00000080:00200000:2.0:1789762970.366842:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001edd2ec9[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 269] 00000080:00200000:2.0:1789762970.366905:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000cb7b0539 17ffffc0001001 3:0 ffff93601a6b02a0 269 lru 00000080:00200000:2.0:1789762970.366980:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000de38e0e7 269: 1< 0 - - > 2< 1101824 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.366990:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001edd2ec9 00000080:00200000:2.0:1789762970.366997:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 14. 00000080:00200000:2.0:1789762970.367007:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 270 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367033:0:16269:0:(rw26.c:871:ll_write_end()) pos 1105920, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367054:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000068ff53dd[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 270] 00000080:00200000:2.0:1789762970.367078:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000942ccd5c 17ffffc0001001 3:0 ffff93601a6b0e00 270 lru 00000080:00200000:2.0:1789762970.367133:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000002baa7a2 270: 1< 0 - - > 2< 1105920 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.367146:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000068ff53dd 00000080:00200000:2.0:1789762970.367153:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 15. 00000080:00200000:2.0:1789762970.367162:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 271 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367185:0:16269:0:(rw26.c:871:ll_write_end()) pos 1110016, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367212:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006ee1d50f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 271] 00000080:00200000:2.0:1789762970.367241:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007f6fda74 17ffffc0001001 3:0 ffff93601a6b01c0 271 lru 00000080:00200000:2.0:1789762970.367308:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000740261ef 271: 1< 0 - - > 2< 1110016 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.367337:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006ee1d50f 00000080:00200000:2.0:1789762970.367344:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 16. 00000080:00200000:2.0:1789762970.367370:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 272 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367394:0:16269:0:(rw26.c:871:ll_write_end()) pos 1114112, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367438:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d61aeb0e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 272] 00000080:00200000:2.0:1789762970.367464:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000064c0f7d 17ffffc0001001 3:0 ffff93601a6b0c40 272 lru 00000080:00200000:2.0:1789762970.367503:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009f86ac45 272: 1< 0 - - > 2< 1114112 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.367515:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d61aeb0e 00000080:00200000:2.0:1789762970.367518:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 17. 00000080:00200000:2.0:1789762970.367524:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 273 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367557:0:16269:0:(rw26.c:871:ll_write_end()) pos 1118208, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367573:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f47988fc[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 273] 00000080:00200000:2.0:1789762970.367604:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e2f45b28 17ffffc0001001 3:0 ffff93601a6b0a80 273 lru 00000080:00200000:2.0:1789762970.367659:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000082db44d8 273: 1< 0 - - > 2< 1118208 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.367683:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f47988fc 00000080:00200000:2.0:1789762970.367685:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 18. 00000080:00200000:2.0:1789762970.367689:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 274 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367715:0:16269:0:(rw26.c:871:ll_write_end()) pos 1122304, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367759:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002623f451[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 274] 00000080:00200000:2.0:1789762970.367791:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000756ee0ec 17ffffc0001001 3:0 ffff93601a6b0700 274 lru 00000080:00200000:2.0:1789762970.367824:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c39fac83 274: 1< 0 - - > 2< 1122304 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.367840:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002623f451 00000080:00200000:2.0:1789762970.367852:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 19. 00000080:00200000:2.0:1789762970.367894:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 275 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.367946:0:16269:0:(rw26.c:871:ll_write_end()) pos 1126400, len 4096, copied 4096 00000080:00200000:2.0:1789762970.367973:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000473a6306[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 275] 00000080:00200000:2.0:1789762970.367981:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000008fc5ada2 17ffffc0001001 3:0 ffff93601a6b08c0 275 lru 00000080:00200000:2.0:1789762970.367996:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006157b157 275: 1< 0 - - > 2< 1126400 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368007:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000473a6306 00000080:00200000:2.0:1789762970.368013:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 20. 00000080:00200000:2.0:1789762970.368025:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 276 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368064:0:16269:0:(rw26.c:871:ll_write_end()) pos 1130496, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368088:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000048bec5b3[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 276] 00000080:00200000:2.0:1789762970.368106:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000da53ad7d 17ffffc0001001 3:0 ffff936008bcd380 276 lru 00000080:00200000:2.0:1789762970.368140:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000587fd882 276: 1< 0 - - > 2< 1130496 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368150:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000048bec5b3 00000080:00200000:2.0:1789762970.368158:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 21. 00000080:00200000:2.0:1789762970.368166:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 277 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368194:0:16269:0:(rw26.c:871:ll_write_end()) pos 1134592, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368236:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000005132986[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 277] 00000080:00200000:2.0:1789762970.368252:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007366d8c3 17ffffc0001021 2:0 ffff936008bcdd20 277 lru 00000080:00200000:2.0:1789762970.368300:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000003a662fc8 277: 1< 0 - - > 2< 1134592 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368315:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000005132986 00000080:00200000:2.0:1789762970.368337:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 22. 00000080:00200000:2.0:1789762970.368363:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 278 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368389:0:16269:0:(rw26.c:871:ll_write_end()) pos 1138688, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368438:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000381e32a7[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 278] 00000080:00200000:2.0:1789762970.368459:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000453f68f1 17ffffc0001001 3:0 ffff936008bcd000 278 lru 00000080:00200000:2.0:1789762970.368493:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000001fd9db1e 278: 1< 0 - - > 2< 1138688 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368503:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000381e32a7 00000080:00200000:2.0:1789762970.368511:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 23. 00000080:00200000:2.0:1789762970.368521:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 279 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368563:0:16269:0:(rw26.c:871:ll_write_end()) pos 1142784, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368597:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007574d371[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 279] 00000080:00200000:2.0:1789762970.368623:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000005d4c4368 17ffffc0001001 3:0 ffff936008bcd2a0 279 lru 00000080:00200000:2.0:1789762970.368685:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f2830544 279: 1< 0 - - > 2< 1142784 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368690:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007574d371 00000080:00200000:2.0:1789762970.368694:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 24. 00000080:00200000:2.0:1789762970.368700:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 280 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368717:0:16269:0:(rw26.c:871:ll_write_end()) pos 1146880, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368755:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000062eb7156[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 280] 00000080:00200000:2.0:1789762970.368789:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006ded22bb 17ffffc0001001 3:0 ffff936008bcde00 280 lru 00000080:00200000:2.0:1789762970.368804:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000349e21cd 280: 1< 0 - - > 2< 1146880 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368807:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000062eb7156 00000080:00200000:2.0:1789762970.368810:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 25. 00000080:00200000:2.0:1789762970.368813:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 281 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.368840:0:16269:0:(rw26.c:871:ll_write_end()) pos 1150976, len 4096, copied 4096 00000080:00200000:2.0:1789762970.368901:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007c6faf41[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 281] 00000080:00200000:2.0:1789762970.368941:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000da5b8799 17ffffc0001001 3:0 ffff936008bcd1c0 281 lru 00000080:00200000:2.0:1789762970.368986:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000013601117 281: 1< 0 - - > 2< 1150976 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.368998:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007c6faf41 00000080:00200000:2.0:1789762970.369003:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 26. 00000080:00200000:2.0:1789762970.369013:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 282 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369037:0:16269:0:(rw26.c:871:ll_write_end()) pos 1155072, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369064:0:16269:0:(rw26.c:929:ll_write_end()) page@000000009218f8b8[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 282] 00000080:00200000:2.0:1789762970.369088:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000efbdf283 17ffffc0001001 3:0 ffff936008bcdc40 282 lru 00000080:00200000:2.0:1789762970.369135:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e4f679a4 282: 1< 0 - - > 2< 1155072 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.369148:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000009218f8b8 00000080:00200000:2.0:1789762970.369157:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 27. 00000080:00200000:2.0:1789762970.369168:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 283 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369192:0:16269:0:(rw26.c:871:ll_write_end()) pos 1159168, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369237:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001483446f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 283] 00000080:00200000:2.0:1789762970.369262:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000024ad4f1f 17ffffc0001001 3:0 ffff936008bcda80 283 lru 00000080:00200000:2.0:1789762970.369311:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006e27769b 283: 1< 0 - - > 2< 1159168 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.369357:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001483446f 00000080:00200000:2.0:1789762970.369363:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 28. 00000080:00200000:2.0:1789762970.369371:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 284 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369400:0:16269:0:(rw26.c:871:ll_write_end()) pos 1163264, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369450:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006f969f95[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 284] 00000080:00200000:2.0:1789762970.369468:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000034f360ef 17ffffc0001001 3:0 ffff936008bcd700 284 lru 00000080:00200000:2.0:1789762970.369488:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006935e899 284: 1< 0 - - > 2< 1163264 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.369499:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006f969f95 00000080:00200000:2.0:1789762970.369507:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 29. 00000080:00200000:2.0:1789762970.369519:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 285 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369555:0:16269:0:(rw26.c:871:ll_write_end()) pos 1167360, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369589:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ffddf4e9[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 285] 00000080:00200000:2.0:1789762970.369609:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000af522b48 17ffffc0001001 3:0 ffff936008bcd8c0 285 lru 00000080:00200000:2.0:1789762970.369672:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000493af35f 285: 1< 0 - - > 2< 1167360 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.369679:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ffddf4e9 00000080:00200000:2.0:1789762970.369685:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 30. 00000080:00200000:2.0:1789762970.369693:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 286 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369718:0:16269:0:(rw26.c:871:ll_write_end()) pos 1171456, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369761:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000075e80cc4[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 286] 00000080:00200000:2.0:1789762970.369787:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f17d5219 17ffffc0001001 3:0 ffff936008bcd7e0 286 lru 00000080:00200000:2.0:1789762970.369808:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000dd305de9 286: 1< 0 - - > 2< 1171456 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.369814:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000075e80cc4 00000080:00200000:2.0:1789762970.369818:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 31. 00000080:00200000:2.0:1789762970.369824:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 287 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.369855:0:16269:0:(rw26.c:871:ll_write_end()) pos 1175552, len 4096, copied 4096 00000080:00200000:2.0:1789762970.369933:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f04cf61a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 287] 00000080:00200000:2.0:1789762970.369949:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000000d28a8b 17ffffc0001001 3:0 ffff936008bcd9a0 287 lru 00000080:00200000:2.0:1789762970.370005:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000239219a3 287: 1< 0 - - > 2< 1175552 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370020:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f04cf61a 00000080:00200000:2.0:1789762970.370030:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 32. 00000080:00200000:2.0:1789762970.370036:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 288 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370068:0:16269:0:(rw26.c:871:ll_write_end()) pos 1179648, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370103:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001aa688a3[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 288] 00000080:00200000:2.0:1789762970.370116:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f0f5cc97 17ffffc0001001 3:0 ffff936008bcd540 288 lru 00000080:00200000:2.0:1789762970.370137:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000083adc8de 288: 1< 0 - - > 2< 1179648 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370152:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001aa688a3 00000080:00200000:2.0:1789762970.370159:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 33. 00000080:00200000:2.0:1789762970.370168:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 289 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370200:0:16269:0:(rw26.c:871:ll_write_end()) pos 1183744, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370245:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005d5215e6[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 289] 00000080:00200000:2.0:1789762970.370254:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006a2431d4 17ffffc0001001 3:0 ffff936008bcdb60 289 lru 00000080:00200000:2.0:1789762970.370259:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000088b31423 289: 1< 0 - - > 2< 1183744 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370261:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005d5215e6 00000080:00200000:2.0:1789762970.370262:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 34. 00000080:00200000:2.0:1789762970.370264:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 290 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370285:0:16269:0:(rw26.c:871:ll_write_end()) pos 1187840, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370335:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000fe1ec36b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 290] 00000080:00200000:2.0:1789762970.370376:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b0564e7c 17ffffc0001001 3:0 ffff936008bcd0e0 290 lru 00000080:00200000:2.0:1789762970.370438:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000002145990b 290: 1< 0 - - > 2< 1187840 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370451:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000fe1ec36b 00000080:00200000:2.0:1789762970.370456:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 35. 00000080:00200000:2.0:1789762970.370464:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 291 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370492:0:16269:0:(rw26.c:871:ll_write_end()) pos 1191936, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370517:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d3d346f0[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 291] 00000080:00200000:2.0:1789762970.370546:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000081cc9fd9 17ffffc0001001 3:0 ffff936008bcd620 291 lru 00000080:00200000:2.0:1789762970.370599:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000027577991 291: 1< 0 - - > 2< 1191936 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370614:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d3d346f0 00000080:00200000:2.0:1789762970.370621:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 36. 00000080:00200000:2.0:1789762970.370632:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 292 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370664:0:16269:0:(rw26.c:871:ll_write_end()) pos 1196032, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370700:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000af4367ec[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 292] 00000080:00200000:2.0:1789762970.370717:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000191fc8dd 17ffffc0001021 2:0 ffff936008bcdee0 292 lru 00000080:00200000:2.0:1789762970.370726:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b776b5f3 292: 1< 0 - - > 2< 1196032 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370755:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000af4367ec 00000080:00200000:2.0:1789762970.370765:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 37. 00000080:00200000:2.0:1789762970.370775:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 293 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370803:0:16269:0:(rw26.c:871:ll_write_end()) pos 1200128, len 4096, copied 4096 00000080:00200000:2.0:1789762970.370811:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f62669d6[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 293] 00000080:00200000:2.0:1789762970.370815:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a42cfb5d 17ffffc0001001 3:0 ffff936008bcd460 293 lru 00000080:00200000:2.0:1789762970.370821:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000871ac269 293: 1< 0 - - > 2< 1200128 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.370824:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f62669d6 00000080:00200000:2.0:1789762970.370831:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 38. 00000080:00200000:2.0:1789762970.370844:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 294 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.370973:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2854 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371004:0:16279:0:(rw26.c:871:ll_write_end()) pos 11689984, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371032:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c98751ad[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2854] 00000080:00200000:2.0:1789762970.371061:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cd1028ef 17ffffc0001001 3:0 ffff936008bcce00 2854 lru 00000080:00200000:2.0:1789762970.371106:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000824611cd 1830: 1< 0 - - > 2< 7495680 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.371122:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c98751ad 00000080:00200000:2.0:1789762970.371133:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1789762970.371146:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2855 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371173:0:16279:0:(rw26.c:871:ll_write_end()) pos 11694080, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371195:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003c29a414[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2855] 00000080:00200000:2.0:1789762970.371233:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000059da9775 17ffffc0001001 3:0 ffff936008bcc1c0 2855 lru 00000080:00200000:2.0:1789762970.371286:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004f8c2048 1831: 1< 0 - - > 2< 7499776 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.371302:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003c29a414 00000080:00200000:2.0:1789762970.371311:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1789762970.371341:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2856 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371394:0:16279:0:(rw26.c:871:ll_write_end()) pos 11698176, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371442:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000066e12522[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2856] 00000080:00200000:2.0:1789762970.371465:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000025900d21 17ffffc0001001 3:0 ffff936008bccc40 2856 lru 00000080:00200000:2.0:1789762970.371512:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007e5eef66 1832: 1< 0 - - > 2< 7503872 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.371526:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000066e12522 00000080:00200000:2.0:1789762970.371531:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1789762970.371536:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2857 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371562:0:16279:0:(rw26.c:871:ll_write_end()) pos 11702272, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371592:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000373f779d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2857] 00000080:00200000:2.0:1789762970.371618:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000085bbf583 17ffffc0001001 3:0 ffff936008bcca80 2857 lru 00000080:00200000:2.0:1789762970.371685:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000779da9f5 1833: 1< 0 - - > 2< 7507968 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.371695:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000373f779d 00000080:00200000:2.0:1789762970.371699:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1789762970.371706:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2858 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371725:0:16279:0:(rw26.c:871:ll_write_end()) pos 11706368, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371761:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a57a0a6a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2858] 00000080:00200000:2.0:1789762970.371782:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003896db7b 17ffffc0001001 3:0 ffff936008bcc700 2858 lru 00000080:00200000:2.0:1789762970.371807:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000001c05485c 1834: 1< 0 - - > 2< 7512064 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.371813:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a57a0a6a 00000080:00200000:2.0:1789762970.371816:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1789762970.371826:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2859 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.371897:0:16279:0:(rw26.c:871:ll_write_end()) pos 11710464, len 4096, copied 4096 00000080:00200000:2.0:1789762970.371939:0:16279:0:(rw26.c:929:ll_write_end()) page@000000004674cf61[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2859] 00000080:00200000:2.0:1789762970.371968:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000eab91fd4 17ffffc0001001 3:0 ffff936008bcc8c0 2859 lru 00000080:00200000:2.0:1789762970.372036:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000b5a3509 1835: 1< 0 - - > 2< 7516160 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372051:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000004674cf61 00000080:00200000:2.0:1789762970.372061:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1789762970.372070:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2860 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.372102:0:16279:0:(rw26.c:871:ll_write_end()) pos 11714560, len 4096, copied 4096 00000080:00200000:2.0:1789762970.372111:0:16279:0:(rw26.c:929:ll_write_end()) page@000000000adf9777[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2860] 00000080:00200000:2.0:1789762970.372129:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000fefc0e15 17ffffc0001001 3:0 ffff936008bcc7e0 2860 lru 00000080:00200000:2.0:1789762970.372186:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003ced7793 1836: 1< 0 - - > 2< 7520256 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372197:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000000adf9777 00000080:00200000:2.0:1789762970.372205:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1789762970.372237:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2861 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.372269:0:16279:0:(rw26.c:871:ll_write_end()) pos 11718656, len 4096, copied 4096 00000080:00200000:2.0:1789762970.372292:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000057f903dc[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2861] 00000080:00200000:2.0:1789762970.372313:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000010c4d4f9 17ffffc0001001 3:0 ffff936008bcc9a0 2861 lru 00000080:00200000:2.0:1789762970.372401:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000002cadb721 1837: 1< 0 - - > 2< 7524352 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372409:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000057f903dc 00000080:00200000:2.0:1789762970.372433:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1789762970.372441:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2862 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.372476:0:16279:0:(rw26.c:871:ll_write_end()) pos 11722752, len 4096, copied 4096 00000080:00200000:2.0:1789762970.372513:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e68ab9e3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2862] 00000080:00200000:2.0:1789762970.372535:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000aa26cc4b 17ffffc0001001 3:0 ffff936008bcc540 2862 lru 00000080:00200000:2.0:1789762970.372579:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a434f393 1838: 1< 0 - - > 2< 7528448 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372585:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e68ab9e3 00000080:00200000:2.0:1789762970.372590:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1789762970.372600:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2863 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.372644:0:16279:0:(rw26.c:871:ll_write_end()) pos 11726848, len 4096, copied 4096 00000080:00200000:2.0:1789762970.372687:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000542db9c2[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2863] 00000080:00200000:2.0:1789762970.372705:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000bdf9274c 17ffffc0001001 3:0 ffff936008bccb60 2863 lru 00000080:00200000:2.0:1789762970.372775:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003bb29c11 1839: 1< 0 - - > 2< 7532544 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372792:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000542db9c2 00000080:00200000:2.0:1789762970.372798:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1789762970.372802:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2864 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.372829:0:16279:0:(rw26.c:871:ll_write_end()) pos 11730944, len 4096, copied 4096 00000080:00200000:2.0:1789762970.372892:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001387c992[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2864] 00000080:00200000:2.0:1789762970.372915:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008ecf9c95 17ffffc0001001 3:0 ffff936008bcc0e0 2864 lru 00000080:00200000:2.0:1789762970.372967:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007968b743 1840: 1< 0 - - > 2< 7536640 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.372975:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001387c992 00000080:00200000:2.0:1789762970.372980:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1789762970.372985:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2865 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373033:0:16279:0:(rw26.c:871:ll_write_end()) pos 11735040, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373062:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008d350380[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2865] 00000080:00200000:2.0:1789762970.373073:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000dc2afb0f 17ffffc0001001 3:0 ffff936008bcc620 2865 lru 00000080:00200000:2.0:1789762970.373125:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d70e0fa1 1841: 1< 0 - - > 2< 7540736 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373137:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008d350380 00000080:00200000:2.0:1789762970.373146:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1789762970.373156:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2866 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373189:0:16279:0:(rw26.c:871:ll_write_end()) pos 11739136, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373215:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c696bd69[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2866] 00000080:00200000:2.0:1789762970.373228:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000044a02c77 17ffffc0001001 3:0 ffff936008bccee0 2866 lru 00000080:00200000:2.0:1789762970.373283:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000006f1b5799 1842: 1< 0 - - > 2< 7544832 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373305:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c696bd69 00000080:00200000:2.0:1789762970.373316:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1789762970.373345:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2867 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373409:0:16279:0:(rw26.c:871:ll_write_end()) pos 11743232, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373442:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003fcb80b9[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2867] 00000080:00200000:2.0:1789762970.373456:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005017d0c3 17ffffc0001021 2:0 ffff936008bcc460 2867 lru 00000080:00200000:2.0:1789762970.373504:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000054d1db73 1843: 1< 0 - - > 2< 7548928 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373513:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003fcb80b9 00000080:00200000:2.0:1789762970.373523:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1789762970.373536:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2868 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373569:0:16279:0:(rw26.c:871:ll_write_end()) pos 11747328, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373592:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ec70759a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2868] 00000080:00200000:2.0:1789762970.373611:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000044049863 17ffffc0001001 3:0 ffff936008bcc380 2868 lru 00000080:00200000:2.0:1789762970.373651:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bc8670a7 1844: 1< 0 - - > 2< 7553024 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373678:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ec70759a 00000080:00200000:2.0:1789762970.373682:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1789762970.373691:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2869 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373718:0:16279:0:(rw26.c:871:ll_write_end()) pos 11751424, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373759:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b161de36[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2869] 00000080:00200000:2.0:1789762970.373774:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000050e75822 17ffffc0001001 3:0 ffff936008bccd20 2869 lru 00000080:00200000:2.0:1789762970.373803:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d2000bc2 1845: 1< 0 - - > 2< 7557120 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373810:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b161de36 00000080:00200000:2.0:1789762970.373814:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1789762970.373820:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2870 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.373885:0:16279:0:(rw26.c:871:ll_write_end()) pos 11755520, len 4096, copied 4096 00000080:00200000:2.0:1789762970.373901:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000f10cb962[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2870] 00000080:00200000:2.0:1789762970.373934:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003155825b 17ffffc0001001 3:0 ffff936008bcc000 2870 lru 00000080:00200000:2.0:1789762970.373972:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f8093068 1846: 1< 0 - - > 2< 7561216 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.373982:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000f10cb962 00000080:00200000:2.0:1789762970.373988:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1789762970.373995:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2871 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374023:0:16279:0:(rw26.c:871:ll_write_end()) pos 11759616, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374041:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d91bcc00[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2871] 00000080:00200000:2.0:1789762970.374065:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005091d5c9 17ffffc0001001 3:0 ffff936008bcc2a0 2871 lru 00000080:00200000:2.0:1789762970.374101:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000739b7373 1847: 1< 0 - - > 2< 7565312 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374112:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d91bcc00 00000080:00200000:2.0:1789762970.374122:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1789762970.374134:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2872 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374171:0:16279:0:(rw26.c:871:ll_write_end()) pos 11763712, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374186:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ab89e8ff[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2872] 00000080:00200000:2.0:1789762970.374205:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c9a4f18d 17ffffc0001001 3:0 ffff93600842fee0 2872 lru 00000080:00200000:2.0:1789762970.374263:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e64d37ef 1848: 1< 0 - - > 2< 7569408 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374273:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ab89e8ff 00000080:00200000:2.0:1789762970.374284:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1789762970.374296:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2873 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374361:0:16279:0:(rw26.c:871:ll_write_end()) pos 11767808, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374393:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000722ae1e7[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2873] 00000080:00200000:2.0:1789762970.374405:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000adc760b5 17ffffc0001001 3:0 ffff93600842f460 2873 lru 00000080:00200000:2.0:1789762970.374451:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000aa2a4f73 1849: 1< 0 - - > 2< 7573504 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374459:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000722ae1e7 00000080:00200000:2.0:1789762970.374466:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1789762970.374476:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2874 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374508:0:16279:0:(rw26.c:871:ll_write_end()) pos 11771904, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374538:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002a345513[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2874] 00000080:00200000:2.0:1789762970.374561:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e2897546 17ffffc0001001 3:0 ffff93600842f380 2874 lru 00000080:00200000:2.0:1789762970.374608:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003679221a 1850: 1< 0 - - > 2< 7577600 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374624:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002a345513 00000080:00200000:2.0:1789762970.374627:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1789762970.374648:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2875 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374670:0:16279:0:(rw26.c:871:ll_write_end()) pos 11776000, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374679:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000cba7e540[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2875] 00000080:00200000:2.0:1789762970.374687:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f10cef2b 17ffffc0001001 3:0 ffff93600842fd20 2875 lru 00000080:00200000:2.0:1789762970.374697:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000178fdb39 1851: 1< 0 - - > 2< 7581696 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374699:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000cba7e540 00000080:00200000:2.0:1789762970.374700:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1789762970.374702:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2876 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374719:0:16279:0:(rw26.c:871:ll_write_end()) pos 11780096, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374745:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000077fd334[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2876] 00000080:00200000:2.0:1789762970.374762:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ebd0815c 17ffffc0001001 3:0 ffff93600842f000 2876 lru 00000080:00200000:2.0:1789762970.374806:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000007e2b2c9 1852: 1< 0 - - > 2< 7585792 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374814:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000077fd334 00000080:00200000:2.0:1789762970.374818:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1789762970.374822:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2877 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.374872:0:16279:0:(rw26.c:871:ll_write_end()) pos 11784192, len 4096, copied 4096 00000080:00200000:2.0:1789762970.374926:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e10ca509[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2877] 00000080:00200000:2.0:1789762970.374943:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000058861983 17ffffc0001001 3:0 ffff93600842f2a0 2877 lru 00000080:00200000:2.0:1789762970.374985:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000512b08df 1853: 1< 0 - - > 2< 7589888 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.374993:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e10ca509 00000080:00200000:2.0:1789762970.374998:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1789762970.375005:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2878 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.375037:0:16279:0:(rw26.c:871:ll_write_end()) pos 11788288, len 4096, copied 4096 00000080:00200000:2.0:1789762970.375051:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009a1d5814[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2878] 00000080:00200000:2.0:1789762970.375064:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000090f95893 17ffffc0001001 3:0 ffff93600842fe00 2878 lru 00000080:00200000:2.0:1789762970.375103:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c8036559 1854: 1< 0 - - > 2< 7593984 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.375116:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009a1d5814 00000080:00200000:2.0:1789762970.375126:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1789762970.375141:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2879 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.375180:0:16279:0:(rw26.c:871:ll_write_end()) pos 11792384, len 4096, copied 4096 00000080:00200000:2.0:1789762970.375203:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c1747d13[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2879] 00000080:00200000:2.0:1789762970.375228:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b87c091b 17ffffc0001001 3:0 ffff93600842f1c0 2879 lru 00000080:00200000:2.0:1789762970.375300:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000002be1f23c 1855: 1< 0 - - > 2< 7598080 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.375316:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c1747d13 00000080:00200000:2.0:1789762970.375337:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1789762970.375353:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2880 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.375403:0:16279:0:(rw26.c:871:ll_write_end()) pos 11796480, len 4096, copied 4096 00000080:00200000:2.0:1789762970.375445:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000383124a4[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2880] 00000080:00200000:2.0:1789762970.375475:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007de07c79 17ffffc0001001 3:0 ffff93600842fc40 2880 lru 00000080:00200000:2.0:1789762970.375537:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000cef9bb10 1856: 1< 0 - - > 2< 7602176 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.375550:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000383124a4 00000080:00200000:2.0:1789762970.375561:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1789762970.375574:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2881 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.375618:0:16279:0:(rw26.c:871:ll_write_end()) pos 11800576, len 4096, copied 4096 00000080:00200000:2.0:1789762970.375649:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000eda73100[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2881] 00000080:00200000:2.0:1789762970.375672:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cc588c9f 17ffffc0001001 3:0 ffff93600842fa80 2881 lru 00000080:00200000:2.0:1789762970.375734:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000fcce81bb 1857: 1< 0 - - > 2< 7606272 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.375745:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000eda73100 00000080:00200000:2.0:1789762970.375753:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1789762970.375766:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2882 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.375853:0:16279:0:(rw26.c:871:ll_write_end()) pos 11804672, len 4096, copied 4096 00000080:00200000:2.0:1789762970.375885:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000002e5dffb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2882] 00000080:00200000:2.0:1789762970.375910:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ee62d6de 17ffffc0001021 2:0 ffff93600842f700 2882 lru 00000080:00200000:2.0:1789762970.375973:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000074668e8a 1858: 1< 0 - - > 2< 7610368 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.375995:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000002e5dffb 00000080:00200000:2.0:1789762970.376002:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1789762970.376010:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2883 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376041:0:16279:0:(rw26.c:871:ll_write_end()) pos 11808768, len 4096, copied 4096 00000080:00200000:2.0:1789762970.376090:0:16279:0:(rw26.c:929:ll_write_end()) page@000000000851afca[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2883] 00000080:00200000:2.0:1789762970.376119:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c94a0061 17ffffc0001001 3:0 ffff93600842f8c0 2883 lru 00000080:00200000:2.0:1789762970.376173:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004ae90f49 1859: 1< 0 - - > 2< 7614464 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.376190:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000000851afca 00000080:00200000:2.0:1789762970.376201:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1789762970.376209:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2884 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376245:0:16279:0:(rw26.c:871:ll_write_end()) pos 11812864, len 4096, copied 4096 00000080:00200000:2.0:1789762970.376285:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d90480fb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2884] 00000080:00200000:2.0:1789762970.376312:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a33ff61f 17ffffc0001001 3:0 ffff93600842f7e0 2884 lru 00000080:00200000:2.0:1789762970.376398:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e3223319 1860: 1< 0 - - > 2< 7618560 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.376411:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d90480fb 00000080:00200000:2.0:1789762970.376422:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1789762970.376428:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2885 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376486:0:16279:0:(rw26.c:871:ll_write_end()) pos 11816960, len 4096, copied 4096 00000080:00200000:2.0:1789762970.376521:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000726c6763[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2885] 00000080:00200000:2.0:1789762970.376550:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f2da9d9d 17ffffc0001001 3:0 ffff93600842f9a0 2885 lru 00000080:00200000:2.0:1789762970.376608:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ebbe521d 1861: 1< 0 - - > 2< 7622656 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.376619:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000726c6763 00000080:00200000:2.0:1789762970.376623:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1789762970.376627:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2886 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376650:0:16279:0:(rw26.c:871:ll_write_end()) pos 11821056, len 4096, copied 4096 00000080:00200000:2.0:1789762970.376676:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000007934d8b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2886] 00000080:00200000:2.0:1789762970.376693:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000160f0626 17ffffc0001001 3:0 ffff93600842f540 2886 lru 00000080:00200000:2.0:1789762970.376740:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003f82bfd9 1862: 1< 0 - - > 2< 7626752 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.376755:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000007934d8b 00000080:00200000:2.0:1789762970.376767:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1789762970.376775:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2887 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376811:0:16279:0:(rw26.c:871:ll_write_end()) pos 11825152, len 4096, copied 4096 00000080:00200000:2.0:1789762970.376819:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ff3f1b48[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2887] 00000080:00200000:2.0:1789762970.376851:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000c47bdb2 17ffffc0001001 3:0 ffff93600842fb60 2887 lru 00000080:00200000:2.0:1789762970.376911:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000009ba251b5 1863: 1< 0 - - > 2< 7630848 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.376926:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ff3f1b48 00000080:00200000:2.0:1789762970.376936:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1789762970.376948:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2888 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.376998:0:16279:0:(rw26.c:871:ll_write_end()) pos 11829248, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377042:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000097da5191[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2888] 00000080:00200000:2.0:1789762970.377064:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008fefb10b 17ffffc0001001 3:0 ffff93600842f0e0 2888 lru 00000080:00200000:2.0:1789762970.377123:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a8bf1232 1864: 1< 0 - - > 2< 7634944 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377138:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000097da5191 00000080:00200000:2.0:1789762970.377152:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1789762970.377169:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2889 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377204:0:16279:0:(rw26.c:871:ll_write_end()) pos 11833344, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377209:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000041af2de2[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2889] 00000080:00200000:2.0:1789762970.377212:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000090681fde 17ffffc0001001 3:0 ffff93600842f620 2889 lru 00000080:00200000:2.0:1789762970.377218:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003f717039 1865: 1< 0 - - > 2< 7639040 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377220:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000041af2de2 00000080:00200000:2.0:1789762970.377222:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1789762970.377225:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2890 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377266:0:16279:0:(rw26.c:871:ll_write_end()) pos 11837440, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377297:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000108d9868[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2890] 00000080:00200000:2.0:1789762970.377340:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000863a5c81 17ffffc0001001 3:0 ffff93600842e000 2890 lru 00000080:00200000:2.0:1789762970.377395:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000005a4dcb8d 1866: 1< 0 - - > 2< 7643136 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377401:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000108d9868 00000080:00200000:2.0:1789762970.377405:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1789762970.377411:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2891 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377441:0:16279:0:(rw26.c:871:ll_write_end()) pos 11841536, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377475:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006376baca[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2891] 00000080:00200000:2.0:1789762970.377495:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000080bd856c 17ffffc0001001 3:0 ffff93600842e2a0 2891 lru 00000080:00200000:2.0:1789762970.377552:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a9279c06 1867: 1< 0 - - > 2< 7647232 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377572:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006376baca 00000080:00200000:2.0:1789762970.377582:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1789762970.377592:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2892 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377620:0:16279:0:(rw26.c:871:ll_write_end()) pos 11845632, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377647:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000009cca002[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2892] 00000080:00200000:2.0:1789762970.377667:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ccfbe6dd 17ffffc0001001 3:0 ffff93600842ee00 2892 lru 00000080:00200000:2.0:1789762970.377706:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000076ac0d57 1868: 1< 0 - - > 2< 7651328 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377719:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000009cca002 00000080:00200000:2.0:1789762970.377734:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1789762970.377752:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2893 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377794:0:16279:0:(rw26.c:871:ll_write_end()) pos 11849728, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377807:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000081b1a18[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2893] 00000080:00200000:2.0:1789762970.377833:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000899f2991 17ffffc0001001 3:0 ffff93600842e1c0 2893 lru 00000080:00200000:2.0:1789762970.377867:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000009c9a3352 1869: 1< 0 - - > 2< 7655424 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.377876:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000081b1a18 00000080:00200000:2.0:1789762970.377877:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1789762970.377880:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2894 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.377904:0:16279:0:(rw26.c:871:ll_write_end()) pos 11853824, len 4096, copied 4096 00000080:00200000:2.0:1789762970.377944:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a57d5b0c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2894] 00000080:00200000:2.0:1789762970.377987:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000af50cbac 17ffffc0001001 3:0 ffff93600842ec40 2894 lru 00000080:00200000:2.0:1789762970.378038:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000822c168c 1870: 1< 0 - - > 2< 7659520 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378048:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a57d5b0c 00000080:00200000:2.0:1789762970.378055:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 79. 00000080:00200000:2.0:1789762970.378065:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2895 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378095:0:16279:0:(rw26.c:871:ll_write_end()) pos 11857920, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378133:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000026e132db[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2895] 00000080:00200000:2.0:1789762970.378159:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000058fc6d47 17ffffc0001001 3:0 ffff93600842ea80 2895 lru 00000080:00200000:2.0:1789762970.378213:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003c85311f 1871: 1< 0 - - > 2< 7663616 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378225:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000026e132db 00000080:00200000:2.0:1789762970.378234:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 80. 00000080:00200000:2.0:1789762970.378250:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2896 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378282:0:16279:0:(rw26.c:871:ll_write_end()) pos 11862016, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378293:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000dc45482b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2896] 00000080:00200000:2.0:1789762970.378300:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000081a451a1 17ffffc0001001 3:0 ffff93600842e700 2896 lru 00000080:00200000:2.0:1789762970.378377:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000e65cae9 1872: 1< 0 - - > 2< 7667712 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378389:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000dc45482b 00000080:00200000:2.0:1789762970.378400:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 81. 00000080:00200000:2.0:1789762970.378412:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2897 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378444:0:16279:0:(rw26.c:871:ll_write_end()) pos 11866112, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378464:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009df97d49[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2897] 00000080:00200000:2.0:1789762970.378482:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001af67294 17ffffc0001021 2:0 ffff93600842e8c0 2897 lru 00000080:00200000:2.0:1789762970.378538:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000000f8cadb 1873: 1< 0 - - > 2< 7671808 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378552:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009df97d49 00000080:00200000:2.0:1789762970.378561:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 82. 00000080:00200000:2.0:1789762970.378574:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2898 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378604:0:16279:0:(rw26.c:871:ll_write_end()) pos 11870208, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378632:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b12d77bb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2898] 00000080:00200000:2.0:1789762970.378652:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a87e7a5f 17ffffc0001001 3:0 ffff93600842e7e0 2898 lru 00000080:00200000:2.0:1789762970.378683:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000102575e4 1874: 1< 0 - - > 2< 7675904 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378693:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b12d77bb 00000080:00200000:2.0:1789762970.378701:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 83. 00000080:00200000:2.0:1789762970.378709:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2899 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378727:0:16279:0:(rw26.c:871:ll_write_end()) pos 11874304, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378759:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008bcf75d8[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2899] 00000080:00200000:2.0:1789762970.378777:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000033143e6f 17ffffc0001001 3:0 ffff93600842e9a0 2899 lru 00000080:00200000:2.0:1789762970.378801:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000001170f527 1875: 1< 0 - - > 2< 7680000 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378806:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008bcf75d8 00000080:00200000:2.0:1789762970.378809:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 84. 00000080:00200000:2.0:1789762970.378831:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2900 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.378852:0:16279:0:(rw26.c:871:ll_write_end()) pos 11878400, len 4096, copied 4096 00000080:00200000:2.0:1789762970.378873:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005940aeb3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2900] 00000080:00200000:2.0:1789762970.378886:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000636ec64f 17ffffc0001001 3:0 ffff93600842e540 2900 lru 00000080:00200000:2.0:1789762970.378975:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a02580ca 1876: 1< 0 - - > 2< 7684096 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.378989:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005940aeb3 00000080:00200000:2.0:1789762970.378996:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 85. 00000080:00200000:2.0:1789762970.379004:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2901 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379046:0:16279:0:(rw26.c:871:ll_write_end()) pos 11882496, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379070:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008b5f60a5[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2901] 00000080:00200000:2.0:1789762970.379095:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cb8cc180 17ffffc0001001 3:0 ffff93600842eb60 2901 lru 00000080:00200000:2.0:1789762970.379177:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d2c38b65 1877: 1< 0 - - > 2< 7688192 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.379190:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008b5f60a5 00000080:00200000:2.0:1789762970.379196:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 86. 00000080:00200000:2.0:1789762970.379201:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2902 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379236:0:16279:0:(rw26.c:871:ll_write_end()) pos 11886592, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379266:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000011a694f3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2902] 00000080:00200000:2.0:1789762970.379300:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000005c36487 17ffffc0001001 3:0 ffff93600842e0e0 2902 lru 00000080:00200000:2.0:1789762970.379370:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000f35544e 1878: 1< 0 - - > 2< 7692288 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.379386:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000011a694f3 00000080:00200000:2.0:1789762970.379391:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 87. 00000080:00200000:2.0:1789762970.379396:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2903 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379426:0:16279:0:(rw26.c:871:ll_write_end()) pos 11890688, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379457:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b1710b11[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2903] 00000080:00200000:2.0:1789762970.379481:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000043a34828 17ffffc0001001 3:0 ffff93600842e620 2903 lru 00000080:00200000:2.0:1789762970.379533:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bbc3bb77 1879: 1< 0 - - > 2< 7696384 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.379544:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b1710b11 00000080:00200000:2.0:1789762970.379551:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 88. 00000080:00200000:2.0:1789762970.379563:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2904 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379594:0:16279:0:(rw26.c:871:ll_write_end()) pos 11894784, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379628:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002f343c2f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2904] 00000080:00200000:2.0:1789762970.379650:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008d63be66 17ffffc0001001 3:0 ffff93600842eee0 2904 lru 00000080:00200000:2.0:1789762970.379720:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000dc4d5073 1880: 1< 0 - - > 2< 7700480 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.379729:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002f343c2f 00000080:00200000:2.0:1789762970.379736:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 89. 00000080:00200000:2.0:1789762970.379747:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2905 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379782:0:16279:0:(rw26.c:871:ll_write_end()) pos 11898880, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379803:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000043140b2f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2905] 00000080:00200000:2.0:1789762970.379812:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008f982513 17ffffc0001001 3:0 ffff93600842e460 2905 lru 00000080:00200000:2.0:1789762970.379864:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000005bfb5505 1881: 1< 0 - - > 2< 7704576 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.379876:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000043140b2f 00000080:00200000:2.0:1789762970.379882:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 90. 00000080:00200000:2.0:1789762970.379886:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2906 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.379919:0:16279:0:(rw26.c:871:ll_write_end()) pos 11902976, len 4096, copied 4096 00000080:00200000:2.0:1789762970.379954:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b4fb8619[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2906] 00000080:00200000:2.0:1789762970.379996:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000062aaa51c 17ffffc0001001 3:0 ffff93600842e380 2906 lru 00000080:00200000:2.0:1789762970.380053:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c35e7786 1882: 1< 0 - - > 2< 7708672 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380059:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b4fb8619 00000080:00200000:2.0:1789762970.380065:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 91. 00000080:00200000:2.0:1789762970.380069:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2907 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.380106:0:16279:0:(rw26.c:871:ll_write_end()) pos 11907072, len 4096, copied 4096 00000080:00200000:2.0:1789762970.380141:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c420649e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2907] 00000080:00200000:2.0:1789762970.380171:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001f017be4 17ffffc0001001 3:0 ffff93600842ed20 2907 lru 00000080:00200000:2.0:1789762970.380206:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000933a27ca 1883: 1< 0 - - > 2< 7712768 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380211:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c420649e 00000080:00200000:2.0:1789762970.380218:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 92. 00000080:00200000:2.0:1789762970.380224:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2908 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.380256:0:16279:0:(rw26.c:871:ll_write_end()) pos 11911168, len 4096, copied 4096 00000080:00200000:2.0:1789762970.380290:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000051e4600c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2908] 00000080:00200000:2.0:1789762970.380312:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cc6261ba 17ffffc0001001 3:0 ffff936010f4a700 2908 lru 00000080:00200000:2.0:1789762970.380376:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008a98b42b 1884: 1< 0 - - > 2< 7716864 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380385:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000051e4600c 00000080:00200000:2.0:1789762970.380392:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 93. 00000080:00200000:2.0:1789762970.380401:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2909 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.380449:0:16279:0:(rw26.c:871:ll_write_end()) pos 11915264, len 4096, copied 4096 00000080:00200000:2.0:1789762970.380474:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000142f9dac[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2909] 00000080:00200000:2.0:1789762970.380497:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008fcf11c7 17ffffc0001001 3:0 ffff936010f4a8c0 2909 lru 00000080:00200000:2.0:1789762970.380551:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000002ccd8bac 1885: 1< 0 - - > 2< 7720960 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380567:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000142f9dac 00000080:00200000:2.0:1789762970.380574:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 94. 00000080:00200000:2.0:1789762970.380585:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2910 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.380613:0:16279:0:(rw26.c:871:ll_write_end()) pos 11919360, len 4096, copied 4096 00000080:00200000:2.0:1789762970.380638:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006fd7aebb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2910] 00000080:00200000:2.0:1789762970.380679:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a1f304de 17ffffc0001001 3:0 ffff936010f4a7e0 2910 lru 00000080:00200000:2.0:1789762970.380757:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008d3eca86 1886: 1< 0 - - > 2< 7725056 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380767:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006fd7aebb 00000080:00200000:2.0:1789762970.380782:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 95. 00000080:00200000:2.0:1789762970.380797:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2911 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.380820:0:16279:0:(rw26.c:871:ll_write_end()) pos 11923456, len 4096, copied 4096 00000080:00200000:2.0:1789762970.380833:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000031d3272b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2911] 00000080:00200000:2.0:1789762970.380838:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004a6e54c1 17ffffc0001001 3:0 ffff936010f4a9a0 2911 lru 00000080:00200000:2.0:1789762970.380910:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000002c7a9858 1887: 1< 0 - - > 2< 7729152 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.380926:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000031d3272b 00000080:00200000:2.0:1789762970.380936:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 96. 00000080:00200000:2.0:1789762970.381004:0:16269:0:(rw26.c:871:ll_write_end()) pos 1204224, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381026:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ebe76f68[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 294] 00000080:00200000:2.0:1789762970.381073:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b3f570e6 17ffffc0001021 2:0 ffff936010f4a540 294 lru 00000080:00200000:2.0:1789762970.381148:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000287df231 294: 1< 0 - - > 2< 1204224 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.381164:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ebe76f68 00000080:00200000:2.0:1789762970.381182:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 39. 00000080:00200000:2.0:1789762970.381206:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 295 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.381259:0:16269:0:(rw26.c:871:ll_write_end()) pos 1208320, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381278:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f3cea8a8[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 295] 00000080:00200000:2.0:1789762970.381285:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000cc6d3801 17ffffc0001001 3:0 ffff936010f4ab60 295 lru 00000080:00200000:2.0:1789762970.381364:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005541c03c 295: 1< 0 - - > 2< 1208320 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.381377:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f3cea8a8 00000080:00200000:2.0:1789762970.381386:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 40. 00000080:00200000:2.0:1789762970.381397:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 296 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.381445:0:16269:0:(rw26.c:871:ll_write_end()) pos 1212416, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381470:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000a38f68f7[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 296] 00000080:00200000:2.0:1789762970.381492:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000012366460 17ffffc0001001 3:0 ffff936010f4a0e0 296 lru 00000080:00200000:2.0:1789762970.381535:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000725a08cd 296: 1< 0 - - > 2< 1212416 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.381550:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000a38f68f7 00000080:00200000:2.0:1789762970.381561:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 41. 00000080:00200000:2.0:1789762970.381569:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 297 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.381619:0:16269:0:(rw26.c:871:ll_write_end()) pos 1216512, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381651:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001b7dc899[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 297] 00000080:00200000:2.0:1789762970.381690:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000054cd1a51 17ffffc0001001 3:0 ffff936010f4a620 297 lru 00000080:00200000:2.0:1789762970.381770:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ea293c4e 297: 1< 0 - - > 2< 1216512 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.381783:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001b7dc899 00000080:00200000:2.0:1789762970.381789:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 42. 00000080:00200000:2.0:1789762970.381797:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 298 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.381818:0:16269:0:(rw26.c:871:ll_write_end()) pos 1220608, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381829:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000030ff6fcf[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 298] 00000080:00200000:2.0:1789762970.381840:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000ec313e64 17ffffc0001001 3:0 ffff936010f4aee0 298 lru 00000080:00200000:2.0:1789762970.381892:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006ad74b87 298: 1< 0 - - > 2< 1220608 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.381901:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000030ff6fcf 00000080:00200000:2.0:1789762970.381907:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 43. 00000080:00200000:2.0:1789762970.381915:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 299 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.381947:0:16269:0:(rw26.c:871:ll_write_end()) pos 1224704, len 4096, copied 4096 00000080:00200000:2.0:1789762970.381982:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d5d1ee4b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 299] 00000080:00200000:2.0:1789762970.382001:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b1274a54 17ffffc0001001 3:0 ffff936010f4a460 299 lru 00000080:00200000:2.0:1789762970.382032:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e1f1ecee 299: 1< 0 - - > 2< 1224704 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382046:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d5d1ee4b 00000080:00200000:2.0:1789762970.382052:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 44. 00000080:00200000:2.0:1789762970.382064:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 300 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382086:0:16269:0:(rw26.c:871:ll_write_end()) pos 1228800, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382119:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008a1e17c8[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 300] 00000080:00200000:2.0:1789762970.382150:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000cd2c0e87 17ffffc0001001 3:0 ffff936010f4a380 300 lru 00000080:00200000:2.0:1789762970.382207:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000aab3f593 300: 1< 0 - - > 2< 1228800 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382215:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008a1e17c8 00000080:00200000:2.0:1789762970.382220:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 45. 00000080:00200000:2.0:1789762970.382228:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 301 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382251:0:16269:0:(rw26.c:871:ll_write_end()) pos 1232896, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382274:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000b81fd018[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 301] 00000080:00200000:2.0:1789762970.382282:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f951e9c6 17ffffc0001001 3:0 ffff936010f4ad20 301 lru 00000080:00200000:2.0:1789762970.382361:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000003c2646c6 301: 1< 0 - - > 2< 1232896 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382370:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000b81fd018 00000080:00200000:2.0:1789762970.382380:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 46. 00000080:00200000:2.0:1789762970.382397:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 302 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382450:0:16269:0:(rw26.c:871:ll_write_end()) pos 1236992, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382476:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005b796b02[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 302] 00000080:00200000:2.0:1789762970.382498:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009b7c3aba 17ffffc0001001 3:0 ffff936010f4a000 302 lru 00000080:00200000:2.0:1789762970.382556:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000749891de 302: 1< 0 - - > 2< 1236992 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382568:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005b796b02 00000080:00200000:2.0:1789762970.382576:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 47. 00000080:00200000:2.0:1789762970.382583:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 303 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382606:0:16269:0:(rw26.c:871:ll_write_end()) pos 1241088, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382627:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008491ca94[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 303] 00000080:00200000:2.0:1789762970.382651:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002b304488 17ffffc0001001 3:0 ffff936010f4a2a0 303 lru 00000080:00200000:2.0:1789762970.382715:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000bdf3d714 303: 1< 0 - - > 2< 1241088 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382722:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008491ca94 00000080:00200000:2.0:1789762970.382728:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 48. 00000080:00200000:2.0:1789762970.382753:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 304 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382784:0:16269:0:(rw26.c:871:ll_write_end()) pos 1245184, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382800:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000cff4f267[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 304] 00000080:00200000:2.0:1789762970.382811:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000eafad4d2 17ffffc0001001 3:0 ffff936010f4ae00 304 lru 00000080:00200000:2.0:1789762970.382845:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005bcbf995 304: 1< 0 - - > 2< 1245184 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.382854:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000cff4f267 00000080:00200000:2.0:1789762970.382887:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 49. 00000080:00200000:2.0:1789762970.382896:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 305 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.382937:0:16269:0:(rw26.c:871:ll_write_end()) pos 1249280, len 4096, copied 4096 00000080:00200000:2.0:1789762970.382955:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c76551ce[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 305] 00000080:00200000:2.0:1789762970.382978:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d58db354 17ffffc0001001 3:0 ffff936010f4a1c0 305 lru 00000080:00200000:2.0:1789762970.383045:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006c411d6c 305: 1< 0 - - > 2< 1249280 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383056:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c76551ce 00000080:00200000:2.0:1789762970.383063:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 50. 00000080:00200000:2.0:1789762970.383072:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 306 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383099:0:16269:0:(rw26.c:871:ll_write_end()) pos 1253376, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383112:0:16269:0:(rw26.c:929:ll_write_end()) page@000000000a2962f1[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 306] 00000080:00200000:2.0:1789762970.383128:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a0886ac9 17ffffc0001001 3:0 ffff936010f4ac40 306 lru 00000080:00200000:2.0:1789762970.383190:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000009bfec7c 306: 1< 0 - - > 2< 1253376 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383202:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000000a2962f1 00000080:00200000:2.0:1789762970.383211:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 51. 00000080:00200000:2.0:1789762970.383220:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 307 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383249:0:16269:0:(rw26.c:871:ll_write_end()) pos 1257472, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383269:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c49dc4eb[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 307] 00000080:00200000:2.0:1789762970.383287:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000098d5875f 17ffffc0001001 3:0 ffff936010f4aa80 307 lru 00000080:00200000:2.0:1789762970.383372:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000761bb7cd 307: 1< 0 - - > 2< 1257472 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383389:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c49dc4eb 00000080:00200000:2.0:1789762970.383397:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 52. 00000080:00200000:2.0:1789762970.383420:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 308 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383471:0:16269:0:(rw26.c:871:ll_write_end()) pos 1261568, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383498:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000079f446d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 308] 00000080:00200000:2.0:1789762970.383507:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000c14f48e6 17ffffc0001001 3:0 ffff936010eb7b60 308 lru 00000080:00200000:2.0:1789762970.383560:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000b4468ff 308: 1< 0 - - > 2< 1261568 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383570:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000079f446d 00000080:00200000:2.0:1789762970.383578:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 53. 00000080:00200000:2.0:1789762970.383588:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 309 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383621:0:16269:0:(rw26.c:871:ll_write_end()) pos 1265664, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383639:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e8ea74ed[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 309] 00000080:00200000:2.0:1789762970.383648:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000030ebfc7d 17ffffc0001021 2:0 ffff936010eb70e0 309 lru 00000080:00200000:2.0:1789762970.383704:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000002727de72 309: 1< 0 - - > 2< 1265664 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383718:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e8ea74ed 00000080:00200000:2.0:1789762970.383725:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 54. 00000080:00200000:2.0:1789762970.383752:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 310 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383786:0:16269:0:(rw26.c:871:ll_write_end()) pos 1269760, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383800:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000062bdbd9e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 310] 00000080:00200000:2.0:1789762970.383805:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000061079ae2 17ffffc0001001 3:0 ffff936010eb7620 310 lru 00000080:00200000:2.0:1789762970.383815:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000007a01916e 310: 1< 0 - - > 2< 1269760 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383821:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000062bdbd9e 00000080:00200000:2.0:1789762970.383823:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 55. 00000080:00200000:2.0:1789762970.383829:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 311 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.383853:0:16269:0:(rw26.c:871:ll_write_end()) pos 1273856, len 4096, copied 4096 00000080:00200000:2.0:1789762970.383895:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002e6ec220[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 311] 00000080:00200000:2.0:1789762970.383911:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000032bd338d 17ffffc0001001 3:0 ffff936010eb7ee0 311 lru 00000080:00200000:2.0:1789762970.383964:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000995c59d4 311: 1< 0 - - > 2< 1273856 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.383978:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002e6ec220 00000080:00200000:2.0:1789762970.383986:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 56. 00000080:00200000:2.0:1789762970.383998:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 312 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384033:0:16269:0:(rw26.c:871:ll_write_end()) pos 1277952, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384059:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000b1517ddf[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 312] 00000080:00200000:2.0:1789762970.384078:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000003a6f95a7 17ffffc0001001 3:0 ffff936010eb7460 312 lru 00000080:00200000:2.0:1789762970.384124:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b47256c7 312: 1< 0 - - > 2< 1277952 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.384142:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000b1517ddf 00000080:00200000:2.0:1789762970.384155:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 57. 00000080:00200000:2.0:1789762970.384163:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 313 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384216:0:16269:0:(rw26.c:871:ll_write_end()) pos 1282048, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384250:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000160633d1[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 313] 00000080:00200000:2.0:1789762970.384282:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000000958f9d 17ffffc0001001 3:0 ffff936010eb7380 313 lru 00000080:00200000:2.0:1789762970.384376:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000029b35148 313: 1< 0 - - > 2< 1282048 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.384390:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000160633d1 00000080:00200000:2.0:1789762970.384400:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 58. 00000080:00200000:2.0:1789762970.384423:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 314 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384451:0:16269:0:(rw26.c:871:ll_write_end()) pos 1286144, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384482:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d5d0ba0b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 314] 00000080:00200000:2.0:1789762970.384503:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002a17b99c 17ffffc0001001 3:0 ffff936010eb7d20 314 lru 00000080:00200000:2.0:1789762970.384566:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006aec442e 314: 1< 0 - - > 2< 1286144 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.384578:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d5d0ba0b 00000080:00200000:2.0:1789762970.384588:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 59. 00000080:00200000:2.0:1789762970.384594:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 315 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384629:0:16269:0:(rw26.c:871:ll_write_end()) pos 1290240, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384636:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000049c875ae[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 315] 00000080:00200000:2.0:1789762970.384657:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a225542c 17ffffc0001001 3:0 ffff936010eb7000 315 lru 00000080:00200000:2.0:1789762970.384734:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000f5060a1 315: 1< 0 - - > 2< 1290240 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.384748:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000049c875ae 00000080:00200000:2.0:1789762970.384760:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 60. 00000080:00200000:2.0:1789762970.384780:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 316 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384798:0:16269:0:(rw26.c:871:ll_write_end()) pos 1294336, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384804:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007781603a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 316] 00000080:00200000:2.0:1789762970.384807:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001b5e5d02 17ffffc0001001 3:0 ffff936010eb72a0 316 lru 00000080:00200000:2.0:1789762970.384811:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f1ec0fca 316: 1< 0 - - > 2< 1294336 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.384814:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007781603a 00000080:00200000:2.0:1789762970.384816:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 61. 00000080:00200000:2.0:1789762970.384818:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 317 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.384876:0:16269:0:(rw26.c:871:ll_write_end()) pos 1298432, len 4096, copied 4096 00000080:00200000:2.0:1789762970.384902:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005f5f6f13[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 317] 00000080:00200000:2.0:1789762970.384946:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000c67e52bf 17ffffc0001001 3:0 ffff936010eb7e00 317 lru 00000080:00200000:2.0:1789762970.385001:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000067512b07 317: 1< 0 - - > 2< 1298432 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385010:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005f5f6f13 00000080:00200000:2.0:1789762970.385013:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 62. 00000080:00200000:2.0:1789762970.385017:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 318 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385052:0:16269:0:(rw26.c:871:ll_write_end()) pos 1302528, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385078:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006fb5b47c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 318] 00000080:00200000:2.0:1789762970.385100:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f38a45aa 17ffffc0001001 3:0 ffff936010eb71c0 318 lru 00000080:00200000:2.0:1789762970.385140:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000004d724e30 318: 1< 0 - - > 2< 1302528 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385149:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006fb5b47c 00000080:00200000:2.0:1789762970.385155:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 63. 00000080:00200000:2.0:1789762970.385165:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 319 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385188:0:16269:0:(rw26.c:871:ll_write_end()) pos 1306624, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385229:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f89bfb17[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 319] 00000080:00200000:2.0:1789762970.385251:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d5f58f78 17ffffc0001001 3:0 ffff936010eb7c40 319 lru 00000080:00200000:2.0:1789762970.385335:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000014956f95 319: 1< 0 - - > 2< 1306624 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385364:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f89bfb17 00000080:00200000:2.0:1789762970.385374:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 64. 00000080:00200000:2.0:1789762970.385383:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 320 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385445:0:16269:0:(rw26.c:871:ll_write_end()) pos 1310720, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385496:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000cd481232[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 320] 00000080:00200000:2.0:1789762970.385527:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000cdb98599 17ffffc0001001 3:0 ffff936010eb7a80 320 lru 00000080:00200000:2.0:1789762970.385589:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000740ac8b5 320: 1< 0 - - > 2< 1310720 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385603:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000cd481232 00000080:00200000:2.0:1789762970.385608:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 65. 00000080:00200000:2.0:1789762970.385611:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 321 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385644:0:16269:0:(rw26.c:871:ll_write_end()) pos 1314816, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385686:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006d7f74a6[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 321] 00000080:00200000:2.0:1789762970.385709:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a118c38a 17ffffc0001001 3:0 ffff936010eb7700 321 lru 00000080:00200000:2.0:1789762970.385743:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000dcd45cd0 321: 1< 0 - - > 2< 1314816 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385752:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006d7f74a6 00000080:00200000:2.0:1789762970.385761:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 66. 00000080:00200000:2.0:1789762970.385769:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 322 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385796:0:16269:0:(rw26.c:871:ll_write_end()) pos 1318912, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385804:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f182c7ae[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 322] 00000080:00200000:2.0:1789762970.385810:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000da540b66 17ffffc0001001 3:0 ffff936010eb78c0 322 lru 00000080:00200000:2.0:1789762970.385839:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c7ab24e3 322: 1< 0 - - > 2< 1318912 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.385863:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f182c7ae 00000080:00200000:2.0:1789762970.385870:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 67. 00000080:00200000:2.0:1789762970.385881:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 323 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.385917:0:16269:0:(rw26.c:871:ll_write_end()) pos 1323008, len 4096, copied 4096 00000080:00200000:2.0:1789762970.385964:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005bacf050[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 323] 00000080:00200000:2.0:1789762970.385989:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e92cd289 17ffffc0001001 3:0 ffff936010eb77e0 323 lru 00000080:00200000:2.0:1789762970.386045:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000192dad17 323: 1< 0 - - > 2< 1323008 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386058:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005bacf050 00000080:00200000:2.0:1789762970.386069:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 68. 00000080:00200000:2.0:1789762970.386078:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 324 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386112:0:16269:0:(rw26.c:871:ll_write_end()) pos 1327104, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386128:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005bc9da20[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 324] 00000080:00200000:2.0:1789762970.386136:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f67e15c4 17ffffc0001021 2:0 ffff936010eb79a0 324 lru 00000080:00200000:2.0:1789762970.386146:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c3aab3fb 324: 1< 0 - - > 2< 1327104 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386165:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005bc9da20 00000080:00200000:2.0:1789762970.386176:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 69. 00000080:00200000:2.0:1789762970.386188:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 325 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386220:0:16269:0:(rw26.c:871:ll_write_end()) pos 1331200, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386242:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c4cb1a32[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 325] 00000080:00200000:2.0:1789762970.386271:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006d591bfb 17ffffc0001001 3:0 ffff936010eb7540 325 lru 00000080:00200000:2.0:1789762970.386332:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000089d750a8 325: 1< 0 - - > 2< 1331200 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386341:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c4cb1a32 00000080:00200000:2.0:1789762970.386370:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 70. 00000080:00200000:2.0:1789762970.386383:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 326 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386438:0:16269:0:(rw26.c:871:ll_write_end()) pos 1335296, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386469:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007ed1fd9f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 326] 00000080:00200000:2.0:1789762970.386492:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000de4e0084 17ffffc0001001 3:0 ffff936006a9bb60 326 lru 00000080:00200000:2.0:1789762970.386532:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b0915f9f 326: 1< 0 - - > 2< 1335296 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386540:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007ed1fd9f 00000080:00200000:2.0:1789762970.386545:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 71. 00000080:00200000:2.0:1789762970.386552:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 327 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386580:0:16269:0:(rw26.c:871:ll_write_end()) pos 1339392, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386603:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e963eaac[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 327] 00000080:00200000:2.0:1789762970.386625:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000020adbfb5 17ffffc0001001 3:0 ffff936006a9b0e0 327 lru 00000080:00200000:2.0:1789762970.386660:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000002a4fdcc1 327: 1< 0 - - > 2< 1339392 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386690:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e963eaac 00000080:00200000:2.0:1789762970.386697:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 72. 00000080:00200000:2.0:1789762970.386703:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 328 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386734:0:16269:0:(rw26.c:871:ll_write_end()) pos 1343488, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386771:0:16269:0:(rw26.c:929:ll_write_end()) page@000000009afcea58[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 328] 00000080:00200000:2.0:1789762970.386795:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000eb5e1815 17ffffc0001001 3:0 ffff936006a9b620 328 lru 00000080:00200000:2.0:1789762970.386809:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000083c197f 328: 1< 0 - - > 2< 1343488 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.386814:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000009afcea58 00000080:00200000:2.0:1789762970.386820:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 73. 00000080:00200000:2.0:1789762970.386826:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 329 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.386883:0:16269:0:(rw26.c:871:ll_write_end()) pos 1347584, len 4096, copied 4096 00000080:00200000:2.0:1789762970.386905:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000bed0c919[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 329] 00000080:00200000:2.0:1789762970.386947:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000005ef9634b 17ffffc0001001 3:0 ffff936006a9bee0 329 lru 00000080:00200000:2.0:1789762970.386997:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000019e0fab5 329: 1< 0 - - > 2< 1347584 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387014:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000bed0c919 00000080:00200000:2.0:1789762970.387022:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 74. 00000080:00200000:2.0:1789762970.387033:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 330 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.387064:0:16269:0:(rw26.c:871:ll_write_end()) pos 1351680, len 4096, copied 4096 00000080:00200000:2.0:1789762970.387085:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008c45a37e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 330] 00000080:00200000:2.0:1789762970.387106:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000081bf9d2d 17ffffc0001001 3:0 ffff936006a9b460 330 lru 00000080:00200000:2.0:1789762970.387120:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000003936d80d 330: 1< 0 - - > 2< 1351680 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387122:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008c45a37e 00000080:00200000:2.0:1789762970.387124:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 75. 00000080:00200000:2.0:1789762970.387125:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 331 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.387153:0:16269:0:(rw26.c:871:ll_write_end()) pos 1355776, len 4096, copied 4096 00000080:00200000:2.0:1789762970.387193:0:16269:0:(rw26.c:929:ll_write_end()) page@000000000778fc20[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 331] 00000080:00200000:2.0:1789762970.387231:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002dbc7622 17ffffc0001001 3:0 ffff936006a9b380 331 lru 00000080:00200000:2.0:1789762970.387298:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000029fe7c90 331: 1< 0 - - > 2< 1355776 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387312:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000000778fc20 00000080:00200000:2.0:1789762970.387333:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 76. 00000080:00200000:2.0:1789762970.387337:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 332 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.387511:0:16269:0:(rw26.c:871:ll_write_end()) pos 1359872, len 4096, copied 4096 00000080:00200000:2.0:1789762970.387543:0:16269:0:(rw26.c:929:ll_write_end()) page@000000009932419b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 332] 00000080:00200000:2.0:1789762970.387572:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006e7ba35f 17ffffc0001001 3:0 ffff936006a9bd20 332 lru 00000080:00200000:2.0:1789762970.387639:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000007fe07d7c 332: 1< 0 - - > 2< 1359872 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387654:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000009932419b 00000080:00200000:2.0:1789762970.387662:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 77. 00000080:00200000:2.0:1789762970.387685:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 333 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.387717:0:16269:0:(rw26.c:871:ll_write_end()) pos 1363968, len 4096, copied 4096 00000080:00200000:2.0:1789762970.387746:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001eee90ca[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 333] 00000080:00200000:2.0:1789762970.387771:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000600adf32 17ffffc0001001 3:0 ffff936006a9b000 333 lru 00000080:00200000:2.0:1789762970.387802:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ec38a7b3 333: 1< 0 - - > 2< 1363968 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387808:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001eee90ca 00000080:00200000:2.0:1789762970.387810:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 78. 00000080:00200000:2.0:1789762970.387835:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 334 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.387854:0:16269:0:(rw26.c:871:ll_write_end()) pos 1368064, len 4096, copied 4096 00000080:00200000:2.0:1789762970.387877:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c9f863cd[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 334] 00000080:00200000:2.0:1789762970.387903:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004126a41e 17ffffc0001001 3:0 ffff936006a9b2a0 334 lru 00000080:00200000:2.0:1789762970.387983:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b512f639 334: 1< 0 - - > 2< 1368064 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.387990:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c9f863cd 00000080:00200000:2.0:1789762970.387995:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 79. 00000080:00200000:2.0:1789762970.388007:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 335 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388040:0:16269:0:(rw26.c:871:ll_write_end()) pos 1372160, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388047:0:16269:0:(rw26.c:929:ll_write_end()) page@000000009a86171a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 335] 00000080:00200000:2.0:1789762970.388052:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000057962b3a 17ffffc0001001 3:0 ffff936006a9be00 335 lru 00000080:00200000:2.0:1789762970.388072:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000852fe321 335: 1< 0 - - > 2< 1372160 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388079:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000009a86171a 00000080:00200000:2.0:1789762970.388085:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 80. 00000080:00200000:2.0:1789762970.388095:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 336 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388126:0:16269:0:(rw26.c:871:ll_write_end()) pos 1376256, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388162:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000093532e1e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 336] 00000080:00200000:2.0:1789762970.388188:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009f6421d3 17ffffc0001001 3:0 ffff936006a9b1c0 336 lru 00000080:00200000:2.0:1789762970.388211:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006660b951 336: 1< 0 - - > 2< 1376256 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388219:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000093532e1e 00000080:00200000:2.0:1789762970.388227:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 81. 00000080:00200000:2.0:1789762970.388237:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 337 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388255:0:16269:0:(rw26.c:871:ll_write_end()) pos 1380352, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388288:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008504e722[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 337] 00000080:00200000:2.0:1789762970.388306:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000daf61521 17ffffc0001001 3:0 ffff936006a9bc40 337 lru 00000080:00200000:2.0:1789762970.388380:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c0293f57 337: 1< 0 - - > 2< 1380352 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388391:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008504e722 00000080:00200000:2.0:1789762970.388398:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 82. 00000080:00200000:2.0:1789762970.388425:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 338 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388448:0:16269:0:(rw26.c:871:ll_write_end()) pos 1384448, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388480:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000013c3c953[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 338] 00000080:00200000:2.0:1789762970.388510:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000bdb78b3e 17ffffc0001001 3:0 ffff936006a9ba80 338 lru 00000080:00200000:2.0:1789762970.388565:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000d501038 338: 1< 0 - - > 2< 1384448 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388572:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000013c3c953 00000080:00200000:2.0:1789762970.388580:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 83. 00000080:00200000:2.0:1789762970.388591:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 339 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388623:0:16269:0:(rw26.c:871:ll_write_end()) pos 1388544, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388637:0:16269:0:(rw26.c:929:ll_write_end()) page@000000000480ac6d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 339] 00000080:00200000:2.0:1789762970.388655:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b3b1fdaf 17ffffc0001021 2:0 ffff936006a9b700 339 lru 00000080:00200000:2.0:1789762970.388720:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000007cc9d393 339: 1< 0 - - > 2< 1388544 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388733:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000000480ac6d 00000080:00200000:2.0:1789762970.388741:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 84. 00000080:00200000:2.0:1789762970.388750:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 340 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388772:0:16269:0:(rw26.c:871:ll_write_end()) pos 1392640, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388796:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000a39e5c81[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 340] 00000080:00200000:2.0:1789762970.388805:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004443e713 17ffffc0001001 3:0 ffff936006a9b8c0 340 lru 00000080:00200000:2.0:1789762970.388815:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000fb2f8094 340: 1< 0 - - > 2< 1392640 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.388819:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000a39e5c81 00000080:00200000:2.0:1789762970.388824:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 85. 00000080:00200000:2.0:1789762970.388830:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 341 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.388882:0:16269:0:(rw26.c:871:ll_write_end()) pos 1396736, len 4096, copied 4096 00000080:00200000:2.0:1789762970.388907:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000b5553852[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 341] 00000080:00200000:2.0:1789762970.388951:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d3063bff 17ffffc0001001 3:0 ffff936006a9b7e0 341 lru 00000080:00200000:2.0:1789762970.388999:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000094ad8ab5 341: 1< 0 - - > 2< 1396736 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389012:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000b5553852 00000080:00200000:2.0:1789762970.389023:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 86. 00000080:00200000:2.0:1789762970.389034:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 342 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.389055:0:16269:0:(rw26.c:871:ll_write_end()) pos 1400832, len 4096, copied 4096 00000080:00200000:2.0:1789762970.389089:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002ed1b917[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 342] 00000080:00200000:2.0:1789762970.389123:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e5e69b25 17ffffc0001001 3:0 ffff936006a9b9a0 342 lru 00000080:00200000:2.0:1789762970.389195:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ec5f7e65 342: 1< 0 - - > 2< 1400832 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389204:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002ed1b917 00000080:00200000:2.0:1789762970.389210:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 87. 00000080:00200000:2.0:1789762970.389222:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 343 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.389256:0:16269:0:(rw26.c:871:ll_write_end()) pos 1404928, len 4096, copied 4096 00000080:00200000:2.0:1789762970.389286:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000514dca29[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 343] 00000080:00200000:2.0:1789762970.389309:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000045c96ec6 17ffffc0001001 3:0 ffff936006a9b540 343 lru 00000080:00200000:2.0:1789762970.389380:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000be60255 343: 1< 0 - - > 2< 1404928 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389395:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000514dca29 00000080:00200000:2.0:1789762970.389407:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 88. 00000080:00200000:2.0:1789762970.389430:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 344 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.389461:0:16269:0:(rw26.c:871:ll_write_end()) pos 1409024, len 4096, copied 4096 00000080:00200000:2.0:1789762970.389492:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000dc48fdaa[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 344] 00000080:00200000:2.0:1789762970.389517:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a4396eab 17ffffc0001001 3:0 ffff936006a9c7e0 344 lru 00000080:00200000:2.0:1789762970.389554:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e3ec4149 344: 1< 0 - - > 2< 1409024 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389564:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000dc48fdaa 00000080:00200000:2.0:1789762970.389572:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 89. 00000080:00200000:2.0:1789762970.389581:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 345 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.389603:0:16269:0:(rw26.c:871:ll_write_end()) pos 1413120, len 4096, copied 4096 00000080:00200000:2.0:1789762970.389627:0:16269:0:(rw26.c:929:ll_write_end()) page@000000003d7b91dc[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 345] 00000080:00200000:2.0:1789762970.389667:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d61f108f 17ffffc0001001 3:0 ffff936006a9c9a0 345 lru 00000080:00200000:2.0:1789762970.389713:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009c57f5c3 345: 1< 0 - - > 2< 1413120 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389723:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000003d7b91dc 00000080:00200000:2.0:1789762970.389730:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 90. 00000080:00200000:2.0:1789762970.389853:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 346 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.389876:0:16269:0:(rw26.c:871:ll_write_end()) pos 1417216, len 4096, copied 4096 00000080:00200000:2.0:1789762970.389915:0:16269:0:(rw26.c:929:ll_write_end()) page@000000000e000beb[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 346] 00000080:00200000:2.0:1789762970.389941:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000893e4054 17ffffc0001001 3:0 ffff936006a9c540 346 lru 00000080:00200000:2.0:1789762970.389988:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000452b4878 346: 1< 0 - - > 2< 1417216 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.389997:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000000e000beb 00000080:00200000:2.0:1789762970.390003:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 91. 00000080:00200000:2.0:1789762970.390007:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 347 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390034:0:16269:0:(rw26.c:871:ll_write_end()) pos 1421312, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390061:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002c1e2824[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 347] 00000080:00200000:2.0:1789762970.390086:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e5a21fca 17ffffc0001001 3:0 ffff936006a9cb60 347 lru 00000080:00200000:2.0:1789762970.390141:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009349a769 347: 1< 0 - - > 2< 1421312 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390152:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002c1e2824 00000080:00200000:2.0:1789762970.390157:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 92. 00000080:00200000:2.0:1789762970.390165:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 348 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390188:0:16269:0:(rw26.c:871:ll_write_end()) pos 1425408, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390222:0:16269:0:(rw26.c:929:ll_write_end()) page@000000003974ac75[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 348] 00000080:00200000:2.0:1789762970.390250:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007dede898 17ffffc0001001 3:0 ffff936006a9c0e0 348 lru 00000080:00200000:2.0:1789762970.390314:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000bcf86f66 348: 1< 0 - - > 2< 1425408 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390337:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000003974ac75 00000080:00200000:2.0:1789762970.390347:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 93. 00000080:00200000:2.0:1789762970.390360:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 349 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390409:0:16269:0:(rw26.c:871:ll_write_end()) pos 1429504, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390439:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f5858adb[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 349] 00000080:00200000:2.0:1789762970.390461:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000875f2e4a 17ffffc0001001 3:0 ffff936006a9c620 349 lru 00000080:00200000:2.0:1789762970.390501:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000aa5543ee 349: 1< 0 - - > 2< 1429504 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390509:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f5858adb 00000080:00200000:2.0:1789762970.390515:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 94. 00000080:00200000:2.0:1789762970.390528:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 350 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390561:0:16269:0:(rw26.c:871:ll_write_end()) pos 1433600, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390592:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000285a6658[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 350] 00000080:00200000:2.0:1789762970.390613:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a25263cd 17ffffc0001001 3:0 ffff936006a9cee0 350 lru 00000080:00200000:2.0:1789762970.390639:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000654c6ee4 350: 1< 0 - - > 2< 1433600 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390644:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000285a6658 00000080:00200000:2.0:1789762970.390647:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 95. 00000080:00200000:2.0:1789762970.390651:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 351 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390686:0:16269:0:(rw26.c:871:ll_write_end()) pos 1437696, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390712:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005be321c8[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 351] 00000080:00200000:2.0:1789762970.390737:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009d22ea47 17ffffc0001001 3:0 ffff936006a9c460 351 lru 00000080:00200000:2.0:1789762970.390777:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000850c222c 351: 1< 0 - - > 2< 1437696 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390790:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005be321c8 00000080:00200000:2.0:1789762970.390799:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 96. 00000080:00200000:2.0:1789762970.390803:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 352 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390824:0:16269:0:(rw26.c:871:ll_write_end()) pos 1441792, len 4096, copied 4096 00000080:00200000:2.0:1789762970.390857:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ddf39152[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 352] 00000080:00200000:2.0:1789762970.390863:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001bf79888 17ffffc0001001 3:0 ffff936006a9c380 352 lru 00000080:00200000:2.0:1789762970.390922:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000fbe0f94c 352: 1< 0 - - > 2< 1441792 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.390935:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ddf39152 00000080:00200000:2.0:1789762970.390947:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 97. 00000080:00200000:2.0:1789762970.390957:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 353 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.390988:0:16269:0:(rw26.c:871:ll_write_end()) pos 1445888, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391014:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f48f2c13[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 353] 00000080:00200000:2.0:1789762970.391027:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f9b8c3f9 17ffffc0001001 3:0 ffff936006a9cd20 353 lru 00000080:00200000:2.0:1789762970.391084:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ffbfe94b 353: 1< 0 - - > 2< 1445888 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391096:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f48f2c13 00000080:00200000:2.0:1789762970.391103:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 98. 00000080:00200000:2.0:1789762970.391112:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 354 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.391137:0:16269:0:(rw26.c:871:ll_write_end()) pos 1449984, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391164:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c3308dfe[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 354] 00000080:00200000:2.0:1789762970.391180:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e822cd10 17ffffc0001021 2:0 ffff936006a9c000 354 lru 00000080:00200000:2.0:1789762970.391234:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000958b6a4 354: 1< 0 - - > 2< 1449984 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391244:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c3308dfe 00000080:00200000:2.0:1789762970.391251:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 99. 00000080:00200000:2.0:1789762970.391264:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 355 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.391290:0:16269:0:(rw26.c:871:ll_write_end()) pos 1454080, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391316:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000020e1e902[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 355] 00000080:00200000:2.0:1789762970.391342:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001328d605 17ffffc0001001 3:0 ffff936006a9c2a0 355 lru 00000080:00200000:2.0:1789762970.391408:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000033d9df8f 355: 1< 0 - - > 2< 1454080 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391426:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000020e1e902 00000080:00200000:2.0:1789762970.391440:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 100. 00000080:00200000:2.0:1789762970.391452:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 356 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.391477:0:16269:0:(rw26.c:871:ll_write_end()) pos 1458176, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391510:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e69d16f2[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 356] 00000080:00200000:2.0:1789762970.391535:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000fa9333df 17ffffc0001001 3:0 ffff936006a9ce00 356 lru 00000080:00200000:2.0:1789762970.391599:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000fae339ec 356: 1< 0 - - > 2< 1458176 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391609:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e69d16f2 00000080:00200000:2.0:1789762970.391615:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 101. 00000080:00200000:2.0:1789762970.391622:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 357 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.391655:0:16269:0:(rw26.c:871:ll_write_end()) pos 1462272, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391692:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004c937d3c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 357] 00000080:00200000:2.0:1789762970.391704:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007669250d 17ffffc0001001 3:0 ffff936006a9c1c0 357 lru 00000080:00200000:2.0:1789762970.391772:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000815d095e 357: 1< 0 - - > 2< 1462272 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391784:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004c937d3c 00000080:00200000:2.0:1789762970.391791:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 102. 00000080:00200000:2.0:1789762970.391799:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 358 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.391818:0:16269:0:(rw26.c:871:ll_write_end()) pos 1466368, len 4096, copied 4096 00000080:00200000:2.0:1789762970.391841:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ff13feb5[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 358] 00000080:00200000:2.0:1789762970.391885:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004a5ff332 17ffffc0001001 3:0 ffff936006a9cc40 358 lru 00000080:00200000:2.0:1789762970.391954:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005b3ec63a 358: 1< 0 - - > 2< 1466368 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.391966:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ff13feb5 00000080:00200000:2.0:1789762970.391975:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 103. 00000080:00200000:2.0:1789762970.391989:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 359 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392035:0:16269:0:(rw26.c:871:ll_write_end()) pos 1470464, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392040:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001819d39e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 359] 00000080:00200000:2.0:1789762970.392042:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000074c8aa4b 17ffffc0001001 3:0 ffff936006a9ca80 359 lru 00000080:00200000:2.0:1789762970.392062:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000db222b22 359: 1< 0 - - > 2< 1470464 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392068:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001819d39e 00000080:00200000:2.0:1789762970.392074:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 104. 00000080:00200000:2.0:1789762970.392112:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2912 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392142:0:16279:0:(rw26.c:871:ll_write_end()) pos 11927552, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392186:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009ab8c57e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2912] 00000080:00200000:2.0:1789762970.392200:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b0615e0a 17ffffc0001001 3:0 ffff936006a9c700 2912 lru 00000080:00200000:2.0:1789762970.392236:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000816517b7 1888: 1< 0 - - > 2< 7733248 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392250:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009ab8c57e 00000080:00200000:2.0:1789762970.392255:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 97. 00000080:00200000:2.0:1789762970.392261:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2913 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392292:0:16279:0:(rw26.c:871:ll_write_end()) pos 11931648, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392313:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005b76b8ce[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2913] 00000080:00200000:2.0:1789762970.392363:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d7edc438 17ffffc0001001 3:0 ffff936006a9c8c0 2913 lru 00000080:00200000:2.0:1789762970.392427:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000002de52b8f 1889: 1< 0 - - > 2< 7737344 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392436:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005b76b8ce 00000080:00200000:2.0:1789762970.392446:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 98. 00000080:00200000:2.0:1789762970.392462:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2914 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392494:0:16279:0:(rw26.c:871:ll_write_end()) pos 11935744, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392525:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000055766478[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2914] 00000080:00200000:2.0:1789762970.392549:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005f7c4bc1 17ffffc0001001 3:0 ffff936006a9d9a0 2914 lru 00000080:00200000:2.0:1789762970.392575:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ffd9bc66 1890: 1< 0 - - > 2< 7741440 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392586:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000055766478 00000080:00200000:2.0:1789762970.392591:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 99. 00000080:00200000:2.0:1789762970.392601:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2915 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392628:0:16279:0:(rw26.c:871:ll_write_end()) pos 11939840, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392659:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b2a0813e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2915] 00000080:00200000:2.0:1789762970.392688:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000fcc7aad9 17ffffc0001001 3:0 ffff936006a9d540 2915 lru 00000080:00200000:2.0:1789762970.392750:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b1f6d38b 1891: 1< 0 - - > 2< 7745536 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392758:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b2a0813e 00000080:00200000:2.0:1789762970.392773:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 100. 00000080:00200000:2.0:1789762970.392791:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2916 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392838:0:16279:0:(rw26.c:871:ll_write_end()) pos 11943936, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392849:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000882bc860[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2916] 00000080:00200000:2.0:1789762970.392859:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000070b5d08d 17ffffc0001001 3:0 ffff936006a9db60 2916 lru 00000080:00200000:2.0:1789762970.392875:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b4ebf35a 1892: 1< 0 - - > 2< 7749632 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.392884:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000882bc860 00000080:00200000:2.0:1789762970.392893:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 101. 00000080:00200000:2.0:1789762970.392905:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2917 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.392944:0:16279:0:(rw26.c:871:ll_write_end()) pos 11948032, len 4096, copied 4096 00000080:00200000:2.0:1789762970.392972:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000028bfc7aa[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2917] 00000080:00200000:2.0:1789762970.392987:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000df486132 17ffffc0001001 3:0 ffff936006a9d0e0 2917 lru 00000080:00200000:2.0:1789762970.393026:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003492dfe3 1893: 1< 0 - - > 2< 7753728 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393031:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000028bfc7aa 00000080:00200000:2.0:1789762970.393036:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 102. 00000080:00200000:2.0:1789762970.393039:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2918 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393062:0:16279:0:(rw26.c:871:ll_write_end()) pos 11952128, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393095:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ef1dff22[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2918] 00000080:00200000:2.0:1789762970.393120:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008c64f21e 17ffffc0001001 3:0 ffff936006a9d620 2918 lru 00000080:00200000:2.0:1789762970.393179:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000622fc4e8 1894: 1< 0 - - > 2< 7757824 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393191:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ef1dff22 00000080:00200000:2.0:1789762970.393202:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 103. 00000080:00200000:2.0:1789762970.393206:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2919 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393232:0:16279:0:(rw26.c:871:ll_write_end()) pos 11956224, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393265:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000088d333a1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2919] 00000080:00200000:2.0:1789762970.393285:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005f1b0515 17ffffc0001001 3:0 ffff936006a9dee0 2919 lru 00000080:00200000:2.0:1789762970.393330:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000076cbeee 1895: 1< 0 - - > 2< 7761920 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393337:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000088d333a1 00000080:00200000:2.0:1789762970.393346:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 104. 00000080:00200000:2.0:1789762970.393356:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2920 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393387:0:16279:0:(rw26.c:871:ll_write_end()) pos 11960320, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393419:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c9f447f8[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2920] 00000080:00200000:2.0:1789762970.393439:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000038fadca 17ffffc0001001 3:0 ffff936006a9d460 2920 lru 00000080:00200000:2.0:1789762970.393475:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000085fab31 1896: 1< 0 - - > 2< 7766016 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393481:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c9f447f8 00000080:00200000:2.0:1789762970.393486:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 105. 00000080:00200000:2.0:1789762970.393494:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2921 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393525:0:16279:0:(rw26.c:871:ll_write_end()) pos 11964416, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393552:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002f05d6c3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2921] 00000080:00200000:2.0:1789762970.393579:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ff467a4f 17ffffc0001021 2:0 ffff936006a9d380 2921 lru 00000080:00200000:2.0:1789762970.393620:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ae5382fd 1897: 1< 0 - - > 2< 7770112 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393631:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002f05d6c3 00000080:00200000:2.0:1789762970.393637:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 106. 00000080:00200000:2.0:1789762970.393641:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2922 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393667:0:16279:0:(rw26.c:871:ll_write_end()) pos 11968512, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393704:0:16279:0:(rw26.c:929:ll_write_end()) page@000000007384598c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2922] 00000080:00200000:2.0:1789762970.393722:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000039fb70d6 17ffffc0001001 3:0 ffff936006a9dd20 2922 lru 00000080:00200000:2.0:1789762970.393758:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000603f1fc7 1898: 1< 0 - - > 2< 7774208 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393766:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000007384598c 00000080:00200000:2.0:1789762970.393773:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 107. 00000080:00200000:2.0:1789762970.393781:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2923 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393834:0:16279:0:(rw26.c:871:ll_write_end()) pos 11972608, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393841:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001f159b81[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2923] 00000080:00200000:2.0:1789762970.393843:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f58928c9 17ffffc0001001 3:0 ffff936006a9d000 2923 lru 00000080:00200000:2.0:1789762970.393847:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000409912a4 1899: 1< 0 - - > 2< 7778304 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393850:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001f159b81 00000080:00200000:2.0:1789762970.393858:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 108. 00000080:00200000:2.0:1789762970.393865:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2924 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.393892:0:16279:0:(rw26.c:871:ll_write_end()) pos 11976704, len 4096, copied 4096 00000080:00200000:2.0:1789762970.393930:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000712fb2c7[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2924] 00000080:00200000:2.0:1789762970.393951:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001bc6cf95 17ffffc0001001 3:0 ffff936006a9d2a0 2924 lru 00000080:00200000:2.0:1789762970.393986:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bef4559a 1900: 1< 0 - - > 2< 7782400 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.393992:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000712fb2c7 00000080:00200000:2.0:1789762970.393995:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 109. 00000080:00200000:2.0:1789762970.394003:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2925 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394026:0:16279:0:(rw26.c:871:ll_write_end()) pos 11980800, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394046:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005aaf7bff[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2925] 00000080:00200000:2.0:1789762970.394065:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000002a6e8c8f 17ffffc0001001 3:0 ffff936006a9de00 2925 lru 00000080:00200000:2.0:1789762970.394076:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000749a889d 1901: 1< 0 - - > 2< 7786496 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394080:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005aaf7bff 00000080:00200000:2.0:1789762970.394084:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 110. 00000080:00200000:2.0:1789762970.394099:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2926 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394127:0:16279:0:(rw26.c:871:ll_write_end()) pos 11984896, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394156:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fab40f89[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2926] 00000080:00200000:2.0:1789762970.394179:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ab149807 17ffffc0001001 3:0 ffff936006a9d1c0 2926 lru 00000080:00200000:2.0:1789762970.394217:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000b783564 1902: 1< 0 - - > 2< 7790592 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394227:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fab40f89 00000080:00200000:2.0:1789762970.394235:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 111. 00000080:00200000:2.0:1789762970.394240:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2927 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394266:0:16279:0:(rw26.c:871:ll_write_end()) pos 11988992, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394294:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009888fe91[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2927] 00000080:00200000:2.0:1789762970.394317:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009b21ed7f 17ffffc0001001 3:0 ffff936006a9dc40 2927 lru 00000080:00200000:2.0:1789762970.394374:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d09a191a 1903: 1< 0 - - > 2< 7794688 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394390:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009888fe91 00000080:00200000:2.0:1789762970.394400:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 112. 00000080:00200000:2.0:1789762970.394411:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2928 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394442:0:16279:0:(rw26.c:871:ll_write_end()) pos 11993088, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394456:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000f547ea8e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2928] 00000080:00200000:2.0:1789762970.394469:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cf07b962 17ffffc0001001 3:0 ffff936006a9da80 2928 lru 00000080:00200000:2.0:1789762970.394493:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000032c5167e 1904: 1< 0 - - > 2< 7798784 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394499:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000f547ea8e 00000080:00200000:2.0:1789762970.394504:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 113. 00000080:00200000:2.0:1789762970.394520:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2929 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394551:0:16279:0:(rw26.c:871:ll_write_end()) pos 11997184, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394582:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002f925c20[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2929] 00000080:00200000:2.0:1789762970.394605:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000093df8087 17ffffc0001001 3:0 ffff936006a9d700 2929 lru 00000080:00200000:2.0:1789762970.394636:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008dd2ab5c 1905: 1< 0 - - > 2< 7802880 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394648:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002f925c20 00000080:00200000:2.0:1789762970.394652:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 114. 00000080:00200000:2.0:1789762970.394656:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2930 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394682:0:16279:0:(rw26.c:871:ll_write_end()) pos 12001280, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394709:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001d36af43[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2930] 00000080:00200000:2.0:1789762970.394731:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d5b0d639 17ffffc0001001 3:0 ffff936006a9d8c0 2930 lru 00000080:00200000:2.0:1789762970.394768:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000070bc41cd 1906: 1< 0 - - > 2< 7806976 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394786:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001d36af43 00000080:00200000:2.0:1789762970.394791:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 115. 00000080:00200000:2.0:1789762970.394839:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2931 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.394861:0:16279:0:(rw26.c:871:ll_write_end()) pos 12005376, len 4096, copied 4096 00000080:00200000:2.0:1789762970.394893:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d8c33221[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2931] 00000080:00200000:2.0:1789762970.394914:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000031150507 17ffffc0001001 3:0 ffff936006a9d7e0 2931 lru 00000080:00200000:2.0:1789762970.394954:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000696842f9 1907: 1< 0 - - > 2< 7811072 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.394966:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d8c33221 00000080:00200000:2.0:1789762970.394975:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 116. 00000080:00200000:2.0:1789762970.394983:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2932 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395020:0:16279:0:(rw26.c:871:ll_write_end()) pos 12009472, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395042:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000bdd5b87f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2932] 00000080:00200000:2.0:1789762970.395070:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000dc99fc3e 17ffffc0001001 3:0 ffff9360082c8620 2932 lru 00000080:00200000:2.0:1789762970.395116:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003a404e73 1908: 1< 0 - - > 2< 7815168 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395128:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000bdd5b87f 00000080:00200000:2.0:1789762970.395137:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 117. 00000080:00200000:2.0:1789762970.395143:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2933 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395173:0:16279:0:(rw26.c:871:ll_write_end()) pos 12013568, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395203:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000044b09280[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2933] 00000080:00200000:2.0:1789762970.395224:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003dbc17ae 17ffffc0001001 3:0 ffff9360082c8ee0 2933 lru 00000080:00200000:2.0:1789762970.395236:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e207e84f 1909: 1< 0 - - > 2< 7819264 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395239:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000044b09280 00000080:00200000:2.0:1789762970.395243:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 118. 00000080:00200000:2.0:1789762970.395253:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2934 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395279:0:16279:0:(rw26.c:871:ll_write_end()) pos 12017664, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395312:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003d1e94c8[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2934] 00000080:00200000:2.0:1789762970.395347:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d1cb5aaa 17ffffc0001001 3:0 ffff9360082c8460 2934 lru 00000080:00200000:2.0:1789762970.395389:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000992113f4 1910: 1< 0 - - > 2< 7823360 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395404:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003d1e94c8 00000080:00200000:2.0:1789762970.395409:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 119. 00000080:00200000:2.0:1789762970.395417:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2935 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395439:0:16279:0:(rw26.c:871:ll_write_end()) pos 12021760, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395471:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000004b9ae6c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2935] 00000080:00200000:2.0:1789762970.395490:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cda67ddf 17ffffc0001001 3:0 ffff9360082c8380 2935 lru 00000080:00200000:2.0:1789762970.395542:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000026fc61b 1911: 1< 0 - - > 2< 7827456 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395555:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000004b9ae6c 00000080:00200000:2.0:1789762970.395565:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 120. 00000080:00200000:2.0:1789762970.395577:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2936 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395600:0:16279:0:(rw26.c:871:ll_write_end()) pos 12025856, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395622:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fe8ed2fa[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2936] 00000080:00200000:2.0:1789762970.395651:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000be2d1981 17ffffc0001021 2:0 ffff9360082c8d20 2936 lru 00000080:00200000:2.0:1789762970.395715:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000005b95b3a4 1912: 1< 0 - - > 2< 7831552 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395726:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fe8ed2fa 00000080:00200000:2.0:1789762970.395735:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 121. 00000080:00200000:2.0:1789762970.395751:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2937 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395834:0:16279:0:(rw26.c:871:ll_write_end()) pos 12029952, len 4096, copied 4096 00000080:00200000:2.0:1789762970.395871:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000510d273a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2937] 00000080:00200000:2.0:1789762970.395891:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000085b1b306 17ffffc0001001 3:0 ffff9360082c8000 2937 lru 00000080:00200000:2.0:1789762970.395935:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000006f36a3ee 1913: 1< 0 - - > 2< 7835648 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.395948:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000510d273a 00000080:00200000:2.0:1789762970.395956:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 122. 00000080:00200000:2.0:1789762970.395965:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2938 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.395997:0:16279:0:(rw26.c:871:ll_write_end()) pos 12034048, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396014:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000047af4d56[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2938] 00000080:00200000:2.0:1789762970.396020:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000074b91dbd 17ffffc0001001 3:0 ffff9360082c82a0 2938 lru 00000080:00200000:2.0:1789762970.396046:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000817ac115 1914: 1< 0 - - > 2< 7839744 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396059:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000047af4d56 00000080:00200000:2.0:1789762970.396067:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 123. 00000080:00200000:2.0:1789762970.396077:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2939 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396101:0:16279:0:(rw26.c:871:ll_write_end()) pos 12038144, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396131:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a5cb4d7e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2939] 00000080:00200000:2.0:1789762970.396158:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000790c674b 17ffffc0001001 3:0 ffff9360082c8e00 2939 lru 00000080:00200000:2.0:1789762970.396189:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c2d951fc 1915: 1< 0 - - > 2< 7843840 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396193:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a5cb4d7e 00000080:00200000:2.0:1789762970.396195:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 124. 00000080:00200000:2.0:1789762970.396198:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2940 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396220:0:16279:0:(rw26.c:871:ll_write_end()) pos 12042240, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396250:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008d98e4da[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2940] 00000080:00200000:2.0:1789762970.396281:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000013e042d7 17ffffc0001001 3:0 ffff9360082c81c0 2940 lru 00000080:00200000:2.0:1789762970.396317:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a05420dd 1916: 1< 0 - - > 2< 7847936 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396338:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008d98e4da 00000080:00200000:2.0:1789762970.396350:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 125. 00000080:00200000:2.0:1789762970.396364:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2941 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396400:0:16279:0:(rw26.c:871:ll_write_end()) pos 12046336, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396434:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000558a3061[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2941] 00000080:00200000:2.0:1789762970.396460:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000bf99947a 17ffffc0001001 3:0 ffff9360082c8c40 2941 lru 00000080:00200000:2.0:1789762970.396529:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000501e74c5 1917: 1< 0 - - > 2< 7852032 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396544:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000558a3061 00000080:00200000:2.0:1789762970.396553:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 126. 00000080:00200000:2.0:1789762970.396567:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2942 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396597:0:16279:0:(rw26.c:871:ll_write_end()) pos 12050432, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396602:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000060c2519c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2942] 00000080:00200000:2.0:1789762970.396625:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000034a7793d 17ffffc0001001 3:0 ffff9360082c8a80 2942 lru 00000080:00200000:2.0:1789762970.396663:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000041a96f7f 1918: 1< 0 - - > 2< 7856128 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396677:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000060c2519c 00000080:00200000:2.0:1789762970.396688:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 127. 00000080:00200000:2.0:1789762970.396696:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2943 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396723:0:16279:0:(rw26.c:871:ll_write_end()) pos 12054528, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396753:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e51ef102[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2943] 00000080:00200000:2.0:1789762970.396766:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b3ede8f4 17ffffc0001001 3:0 ffff9360082c8700 2943 lru 00000080:00200000:2.0:1789762970.396788:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f34ce430 1919: 1< 0 - - > 2< 7860224 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.396832:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e51ef102 00000080:00200000:2.0:1789762970.396840:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 128. 00000080:00200000:2.0:1789762970.396849:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2944 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.396885:0:16279:0:(rw26.c:871:ll_write_end()) pos 12058624, len 4096, copied 4096 00000080:00200000:2.0:1789762970.396923:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b4aca608[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2944] 00000080:00200000:2.0:1789762970.396950:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e137b97a 17ffffc0001001 3:0 ffff9360082c88c0 2944 lru 00000080:00200000:2.0:1789762970.397015:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a448386b 1920: 1< 0 - - > 2< 7864320 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397030:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b4aca608 00000080:00200000:2.0:1789762970.397038:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 129. 00000080:00200000:2.0:1789762970.397049:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2945 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397081:0:16279:0:(rw26.c:871:ll_write_end()) pos 12062720, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397092:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e5852561[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2945] 00000080:00200000:2.0:1789762970.397117:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007e698077 17ffffc0001001 3:0 ffff9360082c87e0 2945 lru 00000080:00200000:2.0:1789762970.397188:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000aa12f914 1921: 1< 0 - - > 2< 7868416 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397199:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e5852561 00000080:00200000:2.0:1789762970.397210:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 130. 00000080:00200000:2.0:1789762970.397221:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2946 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397255:0:16279:0:(rw26.c:871:ll_write_end()) pos 12066816, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397279:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000028117410[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2946] 00000080:00200000:2.0:1789762970.397303:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004b7478b3 17ffffc0001001 3:0 ffff9360082c89a0 2946 lru 00000080:00200000:2.0:1789762970.397371:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000137a956d 1922: 1< 0 - - > 2< 7872512 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397384:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000028117410 00000080:00200000:2.0:1789762970.397397:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 131. 00000080:00200000:2.0:1789762970.397415:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2947 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397445:0:16279:0:(rw26.c:871:ll_write_end()) pos 12070912, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397466:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000031afad1a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2947] 00000080:00200000:2.0:1789762970.397484:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000060d2ae9e 17ffffc0001001 3:0 ffff9360082c8540 2947 lru 00000080:00200000:2.0:1789762970.397531:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003d6379be 1923: 1< 0 - - > 2< 7876608 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397541:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000031afad1a 00000080:00200000:2.0:1789762970.397546:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 132. 00000080:00200000:2.0:1789762970.397553:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2948 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397576:0:16279:0:(rw26.c:871:ll_write_end()) pos 12075008, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397597:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b6fd5aeb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2948] 00000080:00200000:2.0:1789762970.397612:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000095934b02 17ffffc0001001 3:0 ffff9360082c8b60 2948 lru 00000080:00200000:2.0:1789762970.397621:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000020e643a6 1924: 1< 0 - - > 2< 7880704 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397629:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b6fd5aeb 00000080:00200000:2.0:1789762970.397636:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 133. 00000080:00200000:2.0:1789762970.397640:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2949 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397662:0:16279:0:(rw26.c:871:ll_write_end()) pos 12079104, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397688:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000068c1ed53[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2949] 00000080:00200000:2.0:1789762970.397708:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001f004afd 17ffffc0001001 3:0 ffff9360082c80e0 2949 lru 00000080:00200000:2.0:1789762970.397759:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000b398d89 1925: 1< 0 - - > 2< 7884800 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397771:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000068c1ed53 00000080:00200000:2.0:1789762970.397779:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 134. 00000080:00200000:2.0:1789762970.397826:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2950 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.397865:0:16279:0:(rw26.c:871:ll_write_end()) pos 12083200, len 4096, copied 4096 00000080:00200000:2.0:1789762970.397870:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000026fe13e0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2950] 00000080:00200000:2.0:1789762970.397874:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000002415e10a 17ffffc0001001 3:0 ffff936006a9a8c0 2950 lru 00000080:00200000:2.0:1789762970.397929:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e0bd8f18 1926: 1< 0 - - > 2< 7888896 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.397945:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000026fe13e0 00000080:00200000:2.0:1789762970.397954:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 135. 00000080:00200000:2.0:1789762970.397963:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2951 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398007:0:16279:0:(rw26.c:871:ll_write_end()) pos 12087296, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398029:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009fd46778[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2951] 00000080:00200000:2.0:1789762970.398034:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d1b72a9b 17ffffc0001021 2:0 ffff936006a9a7e0 2951 lru 00000080:00200000:2.0:1789762970.398042:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000cbbc619e 1927: 1< 0 - - > 2< 7892992 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398056:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009fd46778 00000080:00200000:2.0:1789762970.398061:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 136. 00000080:00200000:2.0:1789762970.398072:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2952 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398099:0:16279:0:(rw26.c:871:ll_write_end()) pos 12091392, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398131:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000010a4d597[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2952] 00000080:00200000:2.0:1789762970.398151:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000198e1cd 17ffffc0001001 3:0 ffff936006a9a9a0 2952 lru 00000080:00200000:2.0:1789762970.398188:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000049d0d280 1928: 1< 0 - - > 2< 7897088 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398195:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000010a4d597 00000080:00200000:2.0:1789762970.398205:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 137. 00000080:00200000:2.0:1789762970.398212:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2953 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398280:0:16279:0:(rw26.c:871:ll_write_end()) pos 12095488, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398316:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000597f7114[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2953] 00000080:00200000:2.0:1789762970.398357:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000802ab1db 17ffffc0001001 3:0 ffff936006a9a540 2953 lru 00000080:00200000:2.0:1789762970.398430:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b251ccc3 1929: 1< 0 - - > 2< 7901184 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398447:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000597f7114 00000080:00200000:2.0:1789762970.398458:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 138. 00000080:00200000:2.0:1789762970.398470:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2954 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398505:0:16279:0:(rw26.c:871:ll_write_end()) pos 12099584, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398539:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d8636c99[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2954] 00000080:00200000:2.0:1789762970.398561:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009040c924 17ffffc0001001 3:0 ffff936006a9ab60 2954 lru 00000080:00200000:2.0:1789762970.398587:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003979e10d 1930: 1< 0 - - > 2< 7905280 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398603:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d8636c99 00000080:00200000:2.0:1789762970.398611:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 139. 00000080:00200000:2.0:1789762970.398622:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2955 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398660:0:16279:0:(rw26.c:871:ll_write_end()) pos 12103680, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398697:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000058082a1a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2955] 00000080:00200000:2.0:1789762970.398717:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000658b34ee 17ffffc0001001 3:0 ffff936006a9a0e0 2955 lru 00000080:00200000:2.0:1789762970.398751:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003b403b57 1931: 1< 0 - - > 2< 7909376 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398763:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000058082a1a 00000080:00200000:2.0:1789762970.398767:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 140. 00000080:00200000:2.0:1789762970.398775:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2956 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398827:0:16279:0:(rw26.c:871:ll_write_end()) pos 12107776, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398857:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ac099783[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2956] 00000080:00200000:2.0:1789762970.398877:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009b9f3aac 17ffffc0001001 3:0 ffff936006a9a620 2956 lru 00000080:00200000:2.0:1789762970.398908:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004933e9dd 1932: 1< 0 - - > 2< 7913472 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.398922:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ac099783 00000080:00200000:2.0:1789762970.398930:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 141. 00000080:00200000:2.0:1789762970.398944:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2957 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.398971:0:16279:0:(rw26.c:871:ll_write_end()) pos 12111872, len 4096, copied 4096 00000080:00200000:2.0:1789762970.398994:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000077b99528[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2957] 00000080:00200000:2.0:1789762970.399017:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cea6760c 17ffffc0001001 3:0 ffff936006a9aee0 2957 lru 00000080:00200000:2.0:1789762970.399073:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000067d71ea8 1933: 1< 0 - - > 2< 7917568 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399090:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000077b99528 00000080:00200000:2.0:1789762970.399097:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 142. 00000080:00200000:2.0:1789762970.399105:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2958 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399135:0:16279:0:(rw26.c:871:ll_write_end()) pos 12115968, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399164:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000026af8a92[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2958] 00000080:00200000:2.0:1789762970.399179:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008a58597c 17ffffc0001001 3:0 ffff936006a9a460 2958 lru 00000080:00200000:2.0:1789762970.399221:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000061b8855e 1934: 1< 0 - - > 2< 7921664 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399234:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000026af8a92 00000080:00200000:2.0:1789762970.399245:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 143. 00000080:00200000:2.0:1789762970.399259:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2959 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399287:0:16279:0:(rw26.c:871:ll_write_end()) pos 12120064, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399314:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000da2f6731[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2959] 00000080:00200000:2.0:1789762970.399344:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a5a10e2a 17ffffc0001001 3:0 ffff936006a9a380 2959 lru 00000080:00200000:2.0:1789762970.399404:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008156d987 1935: 1< 0 - - > 2< 7925760 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399419:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000da2f6731 00000080:00200000:2.0:1789762970.399426:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 144. 00000080:00200000:2.0:1789762970.399434:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2960 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399462:0:16279:0:(rw26.c:871:ll_write_end()) pos 12124160, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399495:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000018b0a9ec[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2960] 00000080:00200000:2.0:1789762970.399514:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d0b5d57c 17ffffc0001001 3:0 ffff936006a9ad20 2960 lru 00000080:00200000:2.0:1789762970.399560:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000001872bd8e 1936: 1< 0 - - > 2< 7929856 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399568:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000018b0a9ec 00000080:00200000:2.0:1789762970.399574:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 145. 00000080:00200000:2.0:1789762970.399586:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2961 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399623:0:16279:0:(rw26.c:871:ll_write_end()) pos 12128256, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399663:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fcebd098[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2961] 00000080:00200000:2.0:1789762970.399684:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c0a2f20a 17ffffc0001001 3:0 ffff936006a9a000 2961 lru 00000080:00200000:2.0:1789762970.399734:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000045a834fd 1937: 1< 0 - - > 2< 7933952 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399747:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fcebd098 00000080:00200000:2.0:1789762970.399755:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 146. 00000080:00200000:2.0:1789762970.399760:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2962 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399792:0:16279:0:(rw26.c:871:ll_write_end()) pos 12132352, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399850:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000069276713[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2962] 00000080:00200000:2.0:1789762970.399868:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000bbbcd583 17ffffc0001001 3:0 ffff936006a9a2a0 2962 lru 00000080:00200000:2.0:1789762970.399917:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008dcdec1c 1938: 1< 0 - - > 2< 7938048 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.399932:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000069276713 00000080:00200000:2.0:1789762970.399937:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 147. 00000080:00200000:2.0:1789762970.399943:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2963 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.399975:0:16279:0:(rw26.c:871:ll_write_end()) pos 12136448, len 4096, copied 4096 00000080:00200000:2.0:1789762970.399994:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002a4eb992[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2963] 00000080:00200000:2.0:1789762970.400019:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ab801e36 17ffffc0001001 3:0 ffff936006a9ae00 2963 lru 00000080:00200000:2.0:1789762970.400057:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f8b85d1b 1939: 1< 0 - - > 2< 7942144 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400069:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002a4eb992 00000080:00200000:2.0:1789762970.400079:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 148. 00000080:00200000:2.0:1789762970.400084:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2964 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400105:0:16279:0:(rw26.c:871:ll_write_end()) pos 12140544, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400137:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008147e45b[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2964] 00000080:00200000:2.0:1789762970.400160:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b9e2be42 17ffffc0001001 3:0 ffff936006a9a1c0 2964 lru 00000080:00200000:2.0:1789762970.400210:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000267f8277 1940: 1< 0 - - > 2< 7946240 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400224:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008147e45b 00000080:00200000:2.0:1789762970.400229:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 149. 00000080:00200000:2.0:1789762970.400236:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2965 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400268:0:16279:0:(rw26.c:871:ll_write_end()) pos 12144640, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400298:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000192c53b5[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2965] 00000080:00200000:2.0:1789762970.400317:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000ddba40f 17ffffc0001001 3:0 ffff936006a9ac40 2965 lru 00000080:00200000:2.0:1789762970.400381:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bb2d6238 1941: 1< 0 - - > 2< 7950336 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400396:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000192c53b5 00000080:00200000:2.0:1789762970.400405:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 150. 00000080:00200000:2.0:1789762970.400417:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2966 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400452:0:16279:0:(rw26.c:871:ll_write_end()) pos 12148736, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400466:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000070f6fba9[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2966] 00000080:00200000:2.0:1789762970.400491:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000021dc06fa 17ffffc0001021 2:0 ffff936006a9aa80 2966 lru 00000080:00200000:2.0:1789762970.400559:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000caf004e1 1942: 1< 0 - - > 2< 7954432 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400573:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000070f6fba9 00000080:00200000:2.0:1789762970.400580:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 151. 00000080:00200000:2.0:1789762970.400603:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2967 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400639:0:16279:0:(rw26.c:871:ll_write_end()) pos 12152832, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400644:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005d103a5c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2967] 00000080:00200000:2.0:1789762970.400646:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b8f6051a 17ffffc0001001 3:0 ffff936006a9a700 2967 lru 00000080:00200000:2.0:1789762970.400651:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ee2ba5d8 1943: 1< 0 - - > 2< 7958528 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400658:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005d103a5c 00000080:00200000:2.0:1789762970.400665:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 152. 00000080:00200000:2.0:1789762970.400678:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2968 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400709:0:16279:0:(rw26.c:871:ll_write_end()) pos 12156928, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400743:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000040608adc[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2968] 00000080:00200000:2.0:1789762970.400767:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000342a88d1 17ffffc0001001 3:0 ffff936010f48000 2968 lru 00000080:00200000:2.0:1789762970.400822:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000dd0e8e5e 1944: 1< 0 - - > 2< 7962624 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400827:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000040608adc 00000080:00200000:2.0:1789762970.400828:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 153. 00000080:00200000:2.0:1789762970.400833:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2969 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.400863:0:16279:0:(rw26.c:871:ll_write_end()) pos 12161024, len 4096, copied 4096 00000080:00200000:2.0:1789762970.400896:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c9eea40d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2969] 00000080:00200000:2.0:1789762970.400923:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000feaedf1c 17ffffc0001001 3:0 ffff936010f482a0 2969 lru 00000080:00200000:2.0:1789762970.400969:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ea11abd5 1945: 1< 0 - - > 2< 7966720 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.400974:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c9eea40d 00000080:00200000:2.0:1789762970.400980:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 154. 00000080:00200000:2.0:1789762970.400989:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2970 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401015:0:16279:0:(rw26.c:871:ll_write_end()) pos 12165120, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401045:0:16279:0:(rw26.c:929:ll_write_end()) page@000000005a303173[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2970] 00000080:00200000:2.0:1789762970.401067:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003f2356fb 17ffffc0001001 3:0 ffff936010f48e00 2970 lru 00000080:00200000:2.0:1789762970.401137:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b6e8313c 1946: 1< 0 - - > 2< 7970816 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.401153:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000005a303173 00000080:00200000:2.0:1789762970.401163:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 155. 00000080:00200000:2.0:1789762970.401170:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2971 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401213:0:16279:0:(rw26.c:871:ll_write_end()) pos 12169216, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401240:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009205f1bc[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2971] 00000080:00200000:2.0:1789762970.401271:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000097b066e8 17ffffc0001001 3:0 ffff936010f481c0 2971 lru 00000080:00200000:2.0:1789762970.401346:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000090216eef 1947: 1< 0 - - > 2< 7974912 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.401356:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009205f1bc 00000080:00200000:2.0:1789762970.401362:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 156. 00000080:00200000:2.0:1789762970.401374:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2972 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401409:0:16279:0:(rw26.c:871:ll_write_end()) pos 12173312, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401437:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000095971ea6[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2972] 00000080:00200000:2.0:1789762970.401463:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000079e9b587 17ffffc0001001 3:0 ffff936010f48c40 2972 lru 00000080:00200000:2.0:1789762970.401529:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f1aa67a3 1948: 1< 0 - - > 2< 7979008 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.401537:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000095971ea6 00000080:00200000:2.0:1789762970.401548:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 157. 00000080:00200000:2.0:1789762970.401555:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2973 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401575:0:16279:0:(rw26.c:871:ll_write_end()) pos 12177408, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401599:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009dd6da78[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2973] 00000080:00200000:2.0:1789762970.401624:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000055230a 17ffffc0001001 3:0 ffff936010f48a80 2973 lru 00000080:00200000:2.0:1789762970.401674:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004a5079b6 1949: 1< 0 - - > 2< 7983104 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.401686:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009dd6da78 00000080:00200000:2.0:1789762970.401692:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 158. 00000080:00200000:2.0:1789762970.401701:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2974 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401727:0:16279:0:(rw26.c:871:ll_write_end()) pos 12181504, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401757:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000070589ef2[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2974] 00000080:00200000:2.0:1789762970.401775:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003f4c2f0a 17ffffc0001001 3:0 ffff936010f48700 2974 lru 00000080:00200000:2.0:1789762970.401834:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b13cbe5b 1950: 1< 0 - - > 2< 7987200 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.401844:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000070589ef2 00000080:00200000:2.0:1789762970.401852:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 159. 00000080:00200000:2.0:1789762970.401865:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 360 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.401894:0:16269:0:(rw26.c:871:ll_write_end()) pos 1474560, len 4096, copied 4096 00000080:00200000:2.0:1789762970.401921:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000088b4e140[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 360] 00000080:00200000:2.0:1789762970.401935:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000be0c8062 17ffffc0001001 3:0 ffff936010f488c0 360 lru 00000080:00200000:2.0:1789762970.401990:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ae7edb99 360: 1< 0 - - > 2< 1474560 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402002:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000088b4e140 00000080:00200000:2.0:1789762970.402012:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 105. 00000080:00200000:2.0:1789762970.402027:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 361 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402072:0:16269:0:(rw26.c:871:ll_write_end()) pos 1478656, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402083:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001b079a41[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 361] 00000080:00200000:2.0:1789762970.402100:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000092adfeae 17ffffc0001001 3:0 ffff936010f487e0 361 lru 00000080:00200000:2.0:1789762970.402146:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000cd838d30 361: 1< 0 - - > 2< 1478656 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402159:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001b079a41 00000080:00200000:2.0:1789762970.402166:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 106. 00000080:00200000:2.0:1789762970.402178:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 362 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402198:0:16269:0:(rw26.c:871:ll_write_end()) pos 1482752, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402217:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004518129d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 362] 00000080:00200000:2.0:1789762970.402240:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d6549afc 17ffffc0001001 3:0 ffff936010f489a0 362 lru 00000080:00200000:2.0:1789762970.402302:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009bce9e79 362: 1< 0 - - > 2< 1482752 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.402312:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004518129d 00000080:00200000:2.0:1789762970.402329:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 107. 00000080:00200000:2.0:1789762970.402342:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 363 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402370:0:16269:0:(rw26.c:871:ll_write_end()) pos 1486848, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402398:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000065cae0c3[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 363] 00000080:00200000:2.0:1789762970.402422:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002393a552 17ffffc0001001 3:0 ffff936010f48540 363 lru 00000080:00200000:2.0:1789762970.402461:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000003c31d0e0 363: 1< 0 - - > 2< 1486848 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.402471:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000065cae0c3 00000080:00200000:2.0:1789762970.402477:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 108. 00000080:00200000:2.0:1789762970.402483:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 364 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402511:0:16269:0:(rw26.c:871:ll_write_end()) pos 1490944, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402537:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e8cf9560[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 364] 00000080:00200000:2.0:1789762970.402564:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000ec37937c 17ffffc0001001 3:0 ffff936010f48b60 364 lru 00000080:00200000:2.0:1789762970.402608:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f299331a 364: 1< 0 - - > 2< 1490944 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402619:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e8cf9560 00000080:00200000:2.0:1789762970.402630:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 109. 00000080:00200000:2.0:1789762970.402637:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 365 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402656:0:16269:0:(rw26.c:871:ll_write_end()) pos 1495040, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402675:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000014025a8d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 365] 00000080:00200000:2.0:1789762970.402694:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000bfe91b08 17ffffc0001001 3:0 ffff936010f480e0 365 lru 00000080:00200000:2.0:1789762970.402735:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f78d3266 365: 1< 0 - - > 2< 1495040 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402741:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000014025a8d 00000080:00200000:2.0:1789762970.402744:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 110. 00000080:00200000:2.0:1789762970.402751:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 366 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402782:0:16269:0:(rw26.c:871:ll_write_end()) pos 1499136, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402796:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008b437aca[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 366] 00000080:00200000:2.0:1789762970.402825:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000c79678f9 17ffffc0001021 2:0 ffff936010f48620 366 lru 00000080:00200000:2.0:1789762970.402830:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000038b25e7 366: 1< 0 - - > 2< 1499136 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402833:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008b437aca 00000080:00200000:2.0:1789762970.402834:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 111. 00000080:00200000:2.0:1789762970.402838:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 367 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.402855:0:16269:0:(rw26.c:871:ll_write_end()) pos 1503232, len 4096, copied 4096 00000080:00200000:2.0:1789762970.402870:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e0347082[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 367] 00000080:00200000:2.0:1789762970.402897:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000016cfdddc 17ffffc0001001 3:0 ffff936010f48ee0 367 lru 00000080:00200000:2.0:1789762970.402954:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e0e69dc0 367: 1< 0 - - > 2< 1503232 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.402965:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e0347082 00000080:00200000:2.0:1789762970.402976:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 112. 00000080:00200000:2.0:1789762970.402983:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 368 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403012:0:16269:0:(rw26.c:871:ll_write_end()) pos 1507328, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403033:0:16269:0:(rw26.c:929:ll_write_end()) page@000000001848dc0f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 368] 00000080:00200000:2.0:1789762970.403042:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000044a1d6eb 17ffffc0001001 3:0 ffff936010f48460 368 lru 00000080:00200000:2.0:1789762970.403085:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d8a61cac 368: 1< 0 - - > 2< 1507328 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403095:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000001848dc0f 00000080:00200000:2.0:1789762970.403101:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 113. 00000080:00200000:2.0:1789762970.403105:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 369 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403126:0:16269:0:(rw26.c:871:ll_write_end()) pos 1511424, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403141:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000027ae3da7[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 369] 00000080:00200000:2.0:1789762970.403161:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007f3d19ff 17ffffc0001001 3:0 ffff936010f48380 369 lru 00000080:00200000:2.0:1789762970.403210:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c28070fd 369: 1< 0 - - > 2< 1511424 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403226:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000027ae3da7 00000080:00200000:2.0:1789762970.403235:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 114. 00000080:00200000:2.0:1789762970.403251:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 370 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403278:0:16269:0:(rw26.c:871:ll_write_end()) pos 1515520, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403285:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f1e21c14[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 370] 00000080:00200000:2.0:1789762970.403291:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000ae4c29e4 17ffffc0001001 3:0 ffff936010f48d20 370 lru 00000080:00200000:2.0:1789762970.403336:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000cfae2a5f 370: 1< 0 - - > 2< 1515520 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 3407872 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403346:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f1e21c14 00000080:00200000:2.0:1789762970.403354:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 115. 00000080:00200000:2.0:1789762970.403363:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 371 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403405:0:16269:0:(rw26.c:871:ll_write_end()) pos 1519616, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403427:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000165ce24c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 371] 00000080:00200000:2.0:1789762970.403438:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000000a0465f5 17ffffc0001001 3:0 ffff9360110af0e0 371 lru 00000080:00200000:2.0:1789762970.403479:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d0df5170 371: 1< 0 - - > 2< 1519616 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403490:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000165ce24c 00000080:00200000:2.0:1789762970.403499:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 116. 00000080:00200000:2.0:1789762970.403508:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 372 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403534:0:16269:0:(rw26.c:871:ll_write_end()) pos 1523712, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403545:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002168a508[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 372] 00000080:00200000:2.0:1789762970.403547:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004cf25c23 17ffffc0001001 3:0 ffff9360110af620 372 lru 00000080:00200000:2.0:1789762970.403560:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000088ea70ab 372: 1< 0 - - > 2< 1523712 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403565:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002168a508 00000080:00200000:2.0:1789762970.403571:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 117. 00000080:00200000:2.0:1789762970.403577:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 373 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403603:0:16269:0:(rw26.c:871:ll_write_end()) pos 1527808, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403616:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002839c65c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 373] 00000080:00200000:2.0:1789762970.403631:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000030c29fc 17ffffc0001001 3:0 ffff9360110afee0 373 lru 00000080:00200000:2.0:1789762970.403639:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000a345ec2e 373: 1< 0 - - > 2< 1527808 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403641:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002839c65c 00000080:00200000:2.0:1789762970.403643:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 118. 00000080:00200000:2.0:1789762970.403645:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 374 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403661:0:16269:0:(rw26.c:871:ll_write_end()) pos 1531904, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403691:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ee9483be[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 374] 00000080:00200000:2.0:1789762970.403706:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004b146c14 17ffffc0001001 3:0 ffff9360110af460 374 lru 00000080:00200000:2.0:1789762970.403721:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d74d20f8 374: 1< 0 - - > 2< 1531904 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403728:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ee9483be 00000080:00200000:2.0:1789762970.403736:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 119. 00000080:00200000:2.0:1789762970.403741:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 375 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403760:0:16269:0:(rw26.c:871:ll_write_end()) pos 1536000, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403785:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008c7ba46e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 375] 00000080:00200000:2.0:1789762970.403872:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000003657c275 17ffffc0001001 3:0 ffff9360110af380 375 lru 00000080:00200000:2.0:1789762970.403910:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000063e255bf 375: 1< 0 - - > 2< 1536000 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403923:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008c7ba46e 00000080:00200000:2.0:1789762970.403930:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 120. 00000080:00200000:2.0:1789762970.403942:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 376 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.403973:0:16269:0:(rw26.c:871:ll_write_end()) pos 1540096, len 4096, copied 4096 00000080:00200000:2.0:1789762970.403979:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000bb3ac334[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 376] 00000080:00200000:2.0:1789762970.403981:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009041c232 17ffffc0001001 3:0 ffff9360110afd20 376 lru 00000080:00200000:2.0:1789762970.403991:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000baee9b7 376: 1< 0 - - > 2< 1540096 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.403998:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000bb3ac334 00000080:00200000:2.0:1789762970.404008:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 121. 00000080:00200000:2.0:1789762970.404017:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 377 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404044:0:16269:0:(rw26.c:871:ll_write_end()) pos 1544192, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404067:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008dcf7c1f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 377] 00000080:00200000:2.0:1789762970.404078:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e6adcbea 17ffffc0001001 3:0 ffff9360110af000 377 lru 00000080:00200000:2.0:1789762970.404104:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000af135925 377: 1< 0 - - > 2< 1544192 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404111:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008dcf7c1f 00000080:00200000:2.0:1789762970.404115:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 122. 00000080:00200000:2.0:1789762970.404121:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 378 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404148:0:16269:0:(rw26.c:871:ll_write_end()) pos 1548288, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404161:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000fbefbb13[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 378] 00000080:00200000:2.0:1789762970.404181:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009b61d8f9 17ffffc0001001 3:0 ffff9360110af2a0 378 lru 00000080:00200000:2.0:1789762970.404211:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005a393ea2 378: 1< 0 - - > 2< 1548288 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404220:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000fbefbb13 00000080:00200000:2.0:1789762970.404227:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 123. 00000080:00200000:2.0:1789762970.404233:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 379 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404253:0:16269:0:(rw26.c:871:ll_write_end()) pos 1552384, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404269:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000000eca97e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 379] 00000080:00200000:2.0:1789762970.404291:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e61f9cb0 17ffffc0001001 3:0 ffff9360110afe00 379 lru 00000080:00200000:2.0:1789762970.404338:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f660bc8a 379: 1< 0 - - > 2< 1552384 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404345:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000000eca97e 00000080:00200000:2.0:1789762970.404348:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 124. 00000080:00200000:2.0:1789762970.404355:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 380 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404378:0:16269:0:(rw26.c:871:ll_write_end()) pos 1556480, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404387:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000725fbcd2[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 380] 00000080:00200000:2.0:1789762970.404403:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001d2bda3c 17ffffc0001001 3:0 ffff9360110af1c0 380 lru 00000080:00200000:2.0:1789762970.404446:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000069282c22 380: 1< 0 - - > 2< 1556480 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404451:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000725fbcd2 00000080:00200000:2.0:1789762970.404453:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 125. 00000080:00200000:2.0:1789762970.404460:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 381 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404492:0:16269:0:(rw26.c:871:ll_write_end()) pos 1560576, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404498:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000455b4588[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 381] 00000080:00200000:2.0:1789762970.404502:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009ad9348e 17ffffc0001021 2:0 ffff9360110afc40 381 lru 00000080:00200000:2.0:1789762970.404545:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000fe72ad37 381: 1< 0 - - > 2< 1560576 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404556:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000455b4588 00000080:00200000:2.0:1789762970.404563:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 126. 00000080:00200000:2.0:1789762970.404571:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 382 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404596:0:16269:0:(rw26.c:871:ll_write_end()) pos 1564672, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404608:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008f29e07e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 382] 00000080:00200000:2.0:1789762970.404618:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f6e2f70e 17ffffc0001001 3:0 ffff9360110afa80 382 lru 00000080:00200000:2.0:1789762970.404663:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000019c69ea3 382: 1< 0 - - > 2< 1564672 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404675:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008f29e07e 00000080:00200000:2.0:1789762970.404690:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 127. 00000080:00200000:2.0:1789762970.404709:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 383 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404746:0:16269:0:(rw26.c:871:ll_write_end()) pos 1568768, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404780:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000015bdaa27[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 383] 00000080:00200000:2.0:1789762970.404850:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000094b52e3a 17ffffc0001001 3:0 ffff9360110af700 383 lru 00000080:00200000:2.0:1789762970.404886:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000dc12a5ce 383: 1< 0 - - > 2< 1568768 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.404901:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000015bdaa27 00000080:00200000:2.0:1789762970.404909:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 128. 00000080:00200000:2.0:1789762970.404915:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 384 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.404951:0:16269:0:(rw26.c:871:ll_write_end()) pos 1572864, len 4096, copied 4096 00000080:00200000:2.0:1789762970.404962:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000034bec876[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 384] 00000080:00200000:2.0:1789762970.404968:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000007fbe2dd3 17ffffc0001001 3:0 ffff9360110af8c0 384 lru 00000080:00200000:2.0:1789762970.404995:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000bc0172ae 384: 1< 0 - - > 2< 1572864 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405008:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000034bec876 00000080:00200000:2.0:1789762970.405010:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 129. 00000080:00200000:2.0:1789762970.405014:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 385 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405044:0:16269:0:(rw26.c:871:ll_write_end()) pos 1576960, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405064:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000059ad4c31[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 385] 00000080:00200000:2.0:1789762970.405076:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000517cc7b0 17ffffc0001001 3:0 ffff9360110af7e0 385 lru 00000080:00200000:2.0:1789762970.405090:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000723f9874 385: 1< 0 - - > 2< 1576960 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405095:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000059ad4c31 00000080:00200000:2.0:1789762970.405102:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 130. 00000080:00200000:2.0:1789762970.405104:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 386 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405128:0:16269:0:(rw26.c:871:ll_write_end()) pos 1581056, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405139:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006a242f20[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 386] 00000080:00200000:2.0:1789762970.405154:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000027004f64 17ffffc0001001 3:0 ffff9360110af9a0 386 lru 00000080:00200000:2.0:1789762970.405185:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000149ef0cd 386: 1< 0 - - > 2< 1581056 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405192:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006a242f20 00000080:00200000:2.0:1789762970.405198:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 131. 00000080:00200000:2.0:1789762970.405203:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 387 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405224:0:16269:0:(rw26.c:871:ll_write_end()) pos 1585152, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405242:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004ab01635[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 387] 00000080:00200000:2.0:1789762970.405249:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006f8c60bc 17ffffc0001001 3:0 ffff9360110af540 387 lru 00000080:00200000:2.0:1789762970.405253:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000bfcae22e 387: 1< 0 - - > 2< 1585152 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405256:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004ab01635 00000080:00200000:2.0:1789762970.405258:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 132. 00000080:00200000:2.0:1789762970.405261:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 388 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405279:0:16269:0:(rw26.c:871:ll_write_end()) pos 1589248, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405296:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000002f683e[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 388] 00000080:00200000:2.0:1789762970.405313:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000014709f5 17ffffc0001001 3:0 ffff9360110afb60 388 lru 00000080:00200000:2.0:1789762970.405356:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009e72ad05 388: 1< 0 - - > 2< 1589248 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405370:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000002f683e 00000080:00200000:2.0:1789762970.405376:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 133. 00000080:00200000:2.0:1789762970.405389:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 389 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405422:0:16269:0:(rw26.c:871:ll_write_end()) pos 1593344, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405427:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000012f96a81[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 389] 00000080:00200000:2.0:1789762970.405430:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001352fa7c 17ffffc0001001 3:0 ffff9360083449a0 389 lru 00000080:00200000:2.0:1789762970.405435:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000008a1400ef 389: 1< 0 - - > 2< 1593344 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405438:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000012f96a81 00000080:00200000:2.0:1789762970.405447:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 134. 00000080:00200000:2.0:1789762970.405451:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 390 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405472:0:16269:0:(rw26.c:871:ll_write_end()) pos 1597440, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405479:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000cb07adc4[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 390] 00000080:00200000:2.0:1789762970.405487:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e7b2ad55 17ffffc0001001 3:0 ffff936008344540 390 lru 00000080:00200000:2.0:1789762970.405502:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000022432f6e 390: 1< 0 - - > 2< 1597440 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405508:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000cb07adc4 00000080:00200000:2.0:1789762970.405509:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 135. 00000080:00200000:2.0:1789762970.405511:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 391 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405537:0:16269:0:(rw26.c:871:ll_write_end()) pos 1601536, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405553:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004bb6e49c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 391] 00000080:00200000:2.0:1789762970.405568:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f4f1cd20 17ffffc0001001 3:0 ffff936008344b60 391 lru 00000080:00200000:2.0:1789762970.405593:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b862e0e3 391: 1< 0 - - > 2< 1601536 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405604:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004bb6e49c 00000080:00200000:2.0:1789762970.405611:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 136. 00000080:00200000:2.0:1789762970.405617:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 392 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405649:0:16269:0:(rw26.c:871:ll_write_end()) pos 1605632, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405662:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000125f76cb[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 392] 00000080:00200000:2.0:1789762970.405676:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e25285da 17ffffc0001001 3:0 ffff9360083440e0 392 lru 00000080:00200000:2.0:1789762970.405699:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005cda9b6d 392: 1< 0 - - > 2< 1605632 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405704:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000125f76cb 00000080:00200000:2.0:1789762970.405711:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 137. 00000080:00200000:2.0:1789762970.405716:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 393 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405740:0:16269:0:(rw26.c:871:ll_write_end()) pos 1609728, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405770:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000c74876b6[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 393] 00000080:00200000:2.0:1789762970.405787:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000094d2dcba 17ffffc0001001 3:0 ffff936008344620 393 lru 00000080:00200000:2.0:1789762970.405818:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000176c8af5 393: 1< 0 - - > 2< 1609728 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405821:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000c74876b6 00000080:00200000:2.0:1789762970.405823:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 138. 00000080:00200000:2.0:1789762970.405825:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 394 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405846:0:16269:0:(rw26.c:871:ll_write_end()) pos 1613824, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405859:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000013d1394a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 394] 00000080:00200000:2.0:1789762970.405869:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001a90f866 17ffffc0001001 3:0 ffff936008344ee0 394 lru 00000080:00200000:2.0:1789762970.405886:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000cbed0bbd 394: 1< 0 - - > 2< 1613824 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.405890:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000013d1394a 00000080:00200000:2.0:1789762970.405894:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 139. 00000080:00200000:2.0:1789762970.405898:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 395 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.405924:0:16269:0:(rw26.c:871:ll_write_end()) pos 1617920, len 4096, copied 4096 00000080:00200000:2.0:1789762970.405945:0:16269:0:(rw26.c:929:ll_write_end()) page@000000000f48c29f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 395] 00000080:00200000:2.0:1789762970.405963:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000ce2653af 17ffffc0001001 3:0 ffff936008344460 395 lru 00000080:00200000:2.0:1789762970.406004:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000021bf54c9 395: 1< 0 - - > 2< 1617920 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406020:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000000f48c29f 00000080:00200000:2.0:1789762970.406033:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 140. 00000080:00200000:2.0:1789762970.406039:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 396 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406092:0:16269:0:(rw26.c:871:ll_write_end()) pos 1622016, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406114:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008467793a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 396] 00000080:00200000:2.0:1789762970.406125:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000029956e3f 17ffffc0001021 2:0 ffff936008344380 396 lru 00000080:00200000:2.0:1789762970.406173:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000877c7c20 396: 1< 0 - - > 2< 1622016 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406185:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008467793a 00000080:00200000:2.0:1789762970.406191:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 141. 00000080:00200000:2.0:1789762970.406200:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 397 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406226:0:16269:0:(rw26.c:871:ll_write_end()) pos 1626112, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406236:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000046f1748c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 397] 00000080:00200000:2.0:1789762970.406247:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006a1c27b4 17ffffc0001001 3:0 ffff936008344d20 397 lru 00000080:00200000:2.0:1789762970.406264:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000004f42b68c 397: 1< 0 - - > 2< 1626112 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406269:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000046f1748c 00000080:00200000:2.0:1789762970.406271:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 142. 00000080:00200000:2.0:1789762970.406274:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 398 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406293:0:16269:0:(rw26.c:871:ll_write_end()) pos 1630208, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406307:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005a70eaa3[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 398] 00000080:00200000:2.0:1789762970.406316:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000095b11ed1 17ffffc0001001 3:0 ffff936008344000 398 lru 00000080:00200000:2.0:1789762970.406378:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f81f5cc6 398: 1< 0 - - > 2< 1630208 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406390:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005a70eaa3 00000080:00200000:2.0:1789762970.406396:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 143. 00000080:00200000:2.0:1789762970.406409:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 399 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406440:0:16269:0:(rw26.c:871:ll_write_end()) pos 1634304, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406446:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000205e1e66[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 399] 00000080:00200000:2.0:1789762970.406449:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000000f172cae 17ffffc0001001 3:0 ffff9360083442a0 399 lru 00000080:00200000:2.0:1789762970.406471:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000068dcbe43 399: 1< 0 - - > 2< 1634304 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406479:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000205e1e66 00000080:00200000:2.0:1789762970.406484:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 144. 00000080:00200000:2.0:1789762970.406492:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 400 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406511:0:16269:0:(rw26.c:871:ll_write_end()) pos 1638400, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406533:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000b6c00d3a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 400] 00000080:00200000:2.0:1789762970.406545:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f4abcb23 17ffffc0001001 3:0 ffff936008344e00 400 lru 00000080:00200000:2.0:1789762970.406566:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000643b77a 400: 1< 0 - - > 2< 1638400 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406575:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000b6c00d3a 00000080:00200000:2.0:1789762970.406578:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 145. 00000080:00200000:2.0:1789762970.406582:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 401 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406605:0:16269:0:(rw26.c:871:ll_write_end()) pos 1642496, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406632:0:16269:0:(rw26.c:929:ll_write_end()) page@000000002ed8f9e5[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 401] 00000080:00200000:2.0:1789762970.406652:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000042acaef6 17ffffc0001001 3:0 ffff9360083441c0 401 lru 00000080:00200000:2.0:1789762970.406708:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000ca2926e4 401: 1< 0 - - > 2< 1642496 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406723:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000002ed8f9e5 00000080:00200000:2.0:1789762970.406730:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 146. 00000080:00200000:2.0:1789762970.406735:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 402 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406764:0:16269:0:(rw26.c:871:ll_write_end()) pos 1646592, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406782:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d7a055a9[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 402] 00000080:00200000:2.0:1789762970.406816:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e762b4f0 17ffffc0001001 3:0 ffff936008344c40 402 lru 00000080:00200000:2.0:1789762970.406821:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000a8409bb 402: 1< 0 - - > 2< 1646592 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406823:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d7a055a9 00000080:00200000:2.0:1789762970.406824:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 147. 00000080:00200000:2.0:1789762970.406826:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 403 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406844:0:16269:0:(rw26.c:871:ll_write_end()) pos 1650688, len 4096, copied 4096 00000080:00200000:2.0:1789762970.406861:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000ad338591[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 403] 00000080:00200000:2.0:1789762970.406882:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000bb1c8f91 17ffffc0001001 3:0 ffff936008344a80 403 lru 00000080:00200000:2.0:1789762970.406931:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000008f03972a 403: 1< 0 - - > 2< 1650688 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.406943:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000ad338591 00000080:00200000:2.0:1789762970.406949:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 148. 00000080:00200000:2.0:1789762970.406953:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 404 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.406980:0:16269:0:(rw26.c:871:ll_write_end()) pos 1654784, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407002:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000075c9c257[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 404] 00000080:00200000:2.0:1789762970.407026:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000094b5d4a5 17ffffc0001001 3:0 ffff936008344700 404 lru 00000080:00200000:2.0:1789762970.407076:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000002aeab4b2 404: 1< 0 - - > 2< 1654784 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407092:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000075c9c257 00000080:00200000:2.0:1789762970.407099:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 149. 00000080:00200000:2.0:1789762970.407103:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 405 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407123:0:16269:0:(rw26.c:871:ll_write_end()) pos 1658880, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407133:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000803a247d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 405] 00000080:00200000:2.0:1789762970.407150:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000032e4d4ec 17ffffc0001001 3:0 ffff9360083448c0 405 lru 00000080:00200000:2.0:1789762970.407200:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000009cad939e 405: 1< 0 - - > 2< 1658880 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407216:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000803a247d 00000080:00200000:2.0:1789762970.407223:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 150. 00000080:00200000:2.0:1789762970.407232:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 406 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407256:0:16269:0:(rw26.c:871:ll_write_end()) pos 1662976, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407273:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d5268f97[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 406] 00000080:00200000:2.0:1789762970.407282:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e870c2a8 17ffffc0001001 3:0 ffff9360083447e0 406 lru 00000080:00200000:2.0:1789762970.407342:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e411fbf0 406: 1< 0 - - > 2< 1662976 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407351:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d5268f97 00000080:00200000:2.0:1789762970.407357:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 151. 00000080:00200000:2.0:1789762970.407366:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 407 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407408:0:16269:0:(rw26.c:871:ll_write_end()) pos 1667072, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407439:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000135c7951[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 407] 00000080:00200000:2.0:1789762970.407451:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000008525453 17ffffc0001001 3:0 ffff936010f85540 407 lru 00000080:00200000:2.0:1789762970.407485:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000b38af44 407: 1< 0 - - > 2< 1667072 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407499:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000135c7951 00000080:00200000:2.0:1789762970.407504:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 152. 00000080:00200000:2.0:1789762970.407512:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 408 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407533:0:16269:0:(rw26.c:871:ll_write_end()) pos 1671168, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407552:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000051a16a43[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 408] 00000080:00200000:2.0:1789762970.407558:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d74b64ac 17ffffc0001001 3:0 ffff936010f85b60 408 lru 00000080:00200000:2.0:1789762970.407564:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000003f94a9ed 408: 1< 0 - - > 2< 1671168 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407570:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000051a16a43 00000080:00200000:2.0:1789762970.407575:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 153. 00000080:00200000:2.0:1789762970.407583:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 409 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407610:0:16269:0:(rw26.c:871:ll_write_end()) pos 1675264, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407640:0:16269:0:(rw26.c:929:ll_write_end()) page@000000004e599dec[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 409] 00000080:00200000:2.0:1789762970.407663:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000055c3419 17ffffc0001001 3:0 ffff936010f850e0 409 lru 00000080:00200000:2.0:1789762970.407688:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000001734ce27 409: 1< 0 - - > 2< 1675264 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407692:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000004e599dec 00000080:00200000:2.0:1789762970.407697:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 154. 00000080:00200000:2.0:1789762970.407701:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 410 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407721:0:16269:0:(rw26.c:871:ll_write_end()) pos 1679360, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407740:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000271c204a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 410] 00000080:00200000:2.0:1789762970.407760:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004afb1683 17ffffc0001001 3:0 ffff936010f85620 410 lru 00000080:00200000:2.0:1789762970.407794:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f6bea920 410: 1< 0 - - > 2< 1679360 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407825:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000271c204a 00000080:00200000:2.0:1789762970.407828:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 155. 00000080:00200000:2.0:1789762970.407833:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 411 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407854:0:16269:0:(rw26.c:871:ll_write_end()) pos 1683456, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407869:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005254b7ec[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 411] 00000080:00200000:2.0:1789762970.407877:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002594f3cd 17ffffc0001021 2:0 ffff936010f85ee0 411 lru 00000080:00200000:2.0:1789762970.407914:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005777f042 411: 1< 0 - - > 2< 1683456 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.407927:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005254b7ec 00000080:00200000:2.0:1789762970.407938:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 156. 00000080:00200000:2.0:1789762970.407949:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 412 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.407974:0:16269:0:(rw26.c:871:ll_write_end()) pos 1687552, len 4096, copied 4096 00000080:00200000:2.0:1789762970.407984:0:16269:0:(rw26.c:929:ll_write_end()) page@000000005d45d49c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 412] 00000080:00200000:2.0:1789762970.407995:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001c298ab7 17ffffc0001001 3:0 ffff936010f85460 412 lru 00000080:00200000:2.0:1789762970.408037:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006c6b0a1d 412: 1< 0 - - > 2< 1687552 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408047:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000005d45d49c 00000080:00200000:2.0:1789762970.408054:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 157. 00000080:00200000:2.0:1789762970.408060:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 413 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408086:0:16269:0:(rw26.c:871:ll_write_end()) pos 1691648, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408106:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000085b47a84[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 413] 00000080:00200000:2.0:1789762970.408120:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000cf7c926b 17ffffc0001001 3:0 ffff936010f85380 413 lru 00000080:00200000:2.0:1789762970.408152:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d955b389 413: 1< 0 - - > 2< 1691648 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408160:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000085b47a84 00000080:00200000:2.0:1789762970.408165:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 158. 00000080:00200000:2.0:1789762970.408173:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 414 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408206:0:16269:0:(rw26.c:871:ll_write_end()) pos 1695744, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408231:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000555bf5aa[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 414] 00000080:00200000:2.0:1789762970.408256:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000023657c09 17ffffc0001001 3:0 ffff936010f85d20 414 lru 00000080:00200000:2.0:1789762970.408316:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000006b68331a 414: 1< 0 - - > 2< 1695744 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408340:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000555bf5aa 00000080:00200000:2.0:1789762970.408352:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 159. 00000080:00200000:2.0:1789762970.408365:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 415 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408395:0:16269:0:(rw26.c:871:ll_write_end()) pos 1699840, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408418:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000bd5c5e12[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 415] 00000080:00200000:2.0:1789762970.408442:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000002355ac38 17ffffc0001001 3:0 ffff936010f85000 415 lru 00000080:00200000:2.0:1789762970.408503:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000a5467dd2 415: 1< 0 - - > 2< 1699840 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408519:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000bd5c5e12 00000080:00200000:2.0:1789762970.408530:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 160. 00000080:00200000:2.0:1789762970.408542:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 416 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408581:0:16269:0:(rw26.c:871:ll_write_end()) pos 1703936, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408610:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000034bf1c0c[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 416] 00000080:00200000:2.0:1789762970.408632:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a697b19d 17ffffc0001001 3:0 ffff936010f852a0 416 lru 00000080:00200000:2.0:1789762970.408692:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000e3a05386 416: 1< 0 - - > 2< 1703936 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408708:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000034bf1c0c 00000080:00200000:2.0:1789762970.408718:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 161. 00000080:00200000:2.0:1789762970.408731:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 417 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408749:0:16269:0:(rw26.c:871:ll_write_end()) pos 1708032, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408770:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000fa5e0770[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 417] 00000080:00200000:2.0:1789762970.408787:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000009d89af34 17ffffc0001001 3:0 ffff936010f85e00 417 lru 00000080:00200000:2.0:1789762970.408881:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000efc7ce5 417: 1< 0 - - > 2< 1708032 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.408893:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000fa5e0770 00000080:00200000:2.0:1789762970.408904:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 162. 00000080:00200000:2.0:1789762970.408915:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 418 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.408946:0:16269:0:(rw26.c:871:ll_write_end()) pos 1712128, len 4096, copied 4096 00000080:00200000:2.0:1789762970.408975:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000380eadc3[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 418] 00000080:00200000:2.0:1789762970.408994:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000ac6a816c 17ffffc0001001 3:0 ffff936010f851c0 418 lru 00000080:00200000:2.0:1789762970.409056:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000071503957 418: 1< 0 - - > 2< 1712128 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409066:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000380eadc3 00000080:00200000:2.0:1789762970.409073:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 163. 00000080:00200000:2.0:1789762970.409084:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 419 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409108:0:16269:0:(rw26.c:871:ll_write_end()) pos 1716224, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409131:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f13f1644[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 419] 00000080:00200000:2.0:1789762970.409146:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000df735f74 17ffffc0001001 3:0 ffff936010f85c40 419 lru 00000080:00200000:2.0:1789762970.409196:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000008cde84d6 419: 1< 0 - - > 2< 1716224 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409208:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f13f1644 00000080:00200000:2.0:1789762970.409217:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 164. 00000080:00200000:2.0:1789762970.409228:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 420 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409249:0:16269:0:(rw26.c:871:ll_write_end()) pos 1720320, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409273:0:16269:0:(rw26.c:929:ll_write_end()) page@000000009b7a742a[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 420] 00000080:00200000:2.0:1789762970.409297:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000f7a117ed 17ffffc0001001 3:0 ffff936010f85a80 420 lru 00000080:00200000:2.0:1789762970.409336:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d354ee9e 420: 1< 0 - - > 2< 1720320 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409345:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000009b7a742a 00000080:00200000:2.0:1789762970.409357:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 165. 00000080:00200000:2.0:1789762970.409372:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 421 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409401:0:16269:0:(rw26.c:871:ll_write_end()) pos 1724416, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409433:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000070366b64[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 421] 00000080:00200000:2.0:1789762970.409455:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000e30491ef 17ffffc0001001 3:0 ffff936010f85700 421 lru 00000080:00200000:2.0:1789762970.409491:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f8b538ca 421: 1< 0 - - > 2< 1724416 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409502:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000070366b64 00000080:00200000:2.0:1789762970.409509:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 166. 00000080:00200000:2.0:1789762970.409518:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 422 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409545:0:16269:0:(rw26.c:871:ll_write_end()) pos 1728512, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409574:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000e16e3607[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 422] 00000080:00200000:2.0:1789762970.409594:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d93b7f62 17ffffc0001001 3:0 ffff936010f858c0 422 lru 00000080:00200000:2.0:1789762970.409608:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d95c0940 422: 1< 0 - - > 2< 1728512 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409613:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000e16e3607 00000080:00200000:2.0:1789762970.409621:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 167. 00000080:00200000:2.0:1789762970.409630:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 423 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409657:0:16269:0:(rw26.c:871:ll_write_end()) pos 1732608, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409692:0:16269:0:(rw26.c:929:ll_write_end()) page@000000008dc5d1ca[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 423] 00000080:00200000:2.0:1789762970.409709:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000065d6f96b 17ffffc0001001 3:0 ffff936010f857e0 423 lru 00000080:00200000:2.0:1789762970.409762:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000366e80bb 423: 1< 0 - - > 2< 1732608 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409770:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000008dc5d1ca 00000080:00200000:2.0:1789762970.409778:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 168. 00000080:00200000:2.0:1789762970.409786:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 424 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.409865:0:16269:0:(rw26.c:871:ll_write_end()) pos 1736704, len 4096, copied 4096 00000080:00200000:2.0:1789762970.409883:0:16269:0:(rw26.c:929:ll_write_end()) page@000000003d2c49bd[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 424] 00000080:00200000:2.0:1789762970.409892:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000fbf83cac 17ffffc0001001 3:0 ffff936010f859a0 424 lru 00000080:00200000:2.0:1789762970.409927:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000000327484f 424: 1< 0 - - > 2< 1736704 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.409947:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000003d2c49bd 00000080:00200000:2.0:1789762970.409955:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 169. 00000080:00200000:2.0:1789762970.409963:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 425 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410001:0:16269:0:(rw26.c:871:ll_write_end()) pos 1740800, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410016:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000089fab3dd[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 425] 00000080:00200000:2.0:1789762970.410023:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b3be6ee7 17ffffc0001001 3:0 ffff936010d4b8c0 425 lru 00000080:00200000:2.0:1789762970.410060:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@0000000096ed803e 425: 1< 0 - - > 2< 1740800 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410070:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000089fab3dd 00000080:00200000:2.0:1789762970.410078:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 170. 00000080:00200000:2.0:1789762970.410087:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 426 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410117:0:16269:0:(rw26.c:871:ll_write_end()) pos 1744896, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410146:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d35b6e7f[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 426] 00000080:00200000:2.0:1789762970.410164:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000a96b411f 17ffffc0001021 2:0 ffff936010d4b7e0 426 lru 00000080:00200000:2.0:1789762970.410176:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b2325eaa 426: 1< 0 - - > 2< 1744896 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410181:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d35b6e7f 00000080:00200000:2.0:1789762970.410183:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 171. 00000080:00200000:2.0:1789762970.410185:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 427 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410204:0:16269:0:(rw26.c:871:ll_write_end()) pos 1748992, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410251:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000d0209708[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 427] 00000080:00200000:2.0:1789762970.410284:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000003b7e1546 17ffffc0001001 3:0 ffff936010d4b9a0 427 lru 00000080:00200000:2.0:1789762970.410332:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000008f7962eb 427: 1< 0 - - > 2< 1748992 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410342:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000d0209708 00000080:00200000:2.0:1789762970.410349:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 172. 00000080:00200000:2.0:1789762970.410359:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 428 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410392:0:16269:0:(rw26.c:871:ll_write_end()) pos 1753088, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410414:0:16269:0:(rw26.c:929:ll_write_end()) page@0000000094cbde46[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 428] 00000080:00200000:2.0:1789762970.410439:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000013229ff3 17ffffc0001001 3:0 ffff936010d4b540 428 lru 00000080:00200000:2.0:1789762970.410449:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000d18bad6c 428: 1< 0 - - > 2< 1753088 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410459:0:16269:0:(rw26.c:929:ll_write_end()) end page@0000000094cbde46 00000080:00200000:2.0:1789762970.410464:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 173. 00000080:00200000:2.0:1789762970.410476:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 429 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410504:0:16269:0:(rw26.c:871:ll_write_end()) pos 1757184, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410531:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000678ad39d[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 429] 00000080:00200000:2.0:1789762970.410553:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000d0fd763d 17ffffc0001001 3:0 ffff936010d4bb60 429 lru 00000080:00200000:2.0:1789762970.410602:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000f16c2c03 429: 1< 0 - - > 2< 1757184 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410608:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000678ad39d 00000080:00200000:2.0:1789762970.410615:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 174. 00000080:00200000:2.0:1789762970.410624:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 430 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410661:0:16269:0:(rw26.c:871:ll_write_end()) pos 1761280, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410690:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000bd63abaa[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 430] 00000080:00200000:2.0:1789762970.410701:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000006057b988 17ffffc0001001 3:0 ffff936010d4b0e0 430 lru 00000080:00200000:2.0:1789762970.410749:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000dd7e3625 430: 1< 0 - - > 2< 1761280 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410759:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000bd63abaa 00000080:00200000:2.0:1789762970.410768:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 175. 00000080:00200000:2.0:1789762970.410781:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 431 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.410858:0:16269:0:(rw26.c:871:ll_write_end()) pos 1765376, len 4096, copied 4096 00000080:00200000:2.0:1789762970.410879:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000bc906cdb[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 431] 00000080:00200000:2.0:1789762970.410897:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000c2c0088a 17ffffc0001001 3:0 ffff936010d4b620 431 lru 00000080:00200000:2.0:1789762970.410948:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000b359b7e2 431: 1< 0 - - > 2< 1765376 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.410954:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000bc906cdb 00000080:00200000:2.0:1789762970.410962:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 176. 00000080:00200000:2.0:1789762970.410974:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 432 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411007:0:16269:0:(rw26.c:871:ll_write_end()) pos 1769472, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411040:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006d94aecc[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 432] 00000080:00200000:2.0:1789762970.411059:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000001c5a9f7a 17ffffc0001001 3:0 ffff936010d4bee0 432 lru 00000080:00200000:2.0:1789762970.411115:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000889fd208 432: 1< 0 - - > 2< 1769472 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411130:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006d94aecc 00000080:00200000:2.0:1789762970.411139:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 177. 00000080:00200000:2.0:1789762970.411151:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 433 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411181:0:16269:0:(rw26.c:871:ll_write_end()) pos 1773568, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411219:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007e135f74[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 433] 00000080:00200000:2.0:1789762970.411236:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @000000004fe6ce87 17ffffc0001001 3:0 ffff936010d4b460 433 lru 00000080:00200000:2.0:1789762970.411298:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000cdbab868 433: 1< 0 - - > 2< 1773568 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411311:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007e135f74 00000080:00200000:2.0:1789762970.411333:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 178. 00000080:00200000:2.0:1789762970.411345:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 434 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411368:0:16269:0:(rw26.c:871:ll_write_end()) pos 1777664, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411396:0:16269:0:(rw26.c:929:ll_write_end()) page@000000006744fe1b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 434] 00000080:00200000:2.0:1789762970.411419:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000086b51184 17ffffc0001001 3:0 ffff936010d4b380 434 lru 00000080:00200000:2.0:1789762970.411449:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000c0c69dcd 434: 1< 0 - - > 2< 1777664 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411460:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000006744fe1b 00000080:00200000:2.0:1789762970.411463:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 179. 00000080:00200000:2.0:1789762970.411470:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 435 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411497:0:16269:0:(rw26.c:871:ll_write_end()) pos 1781760, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411515:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000088e288b[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 435] 00000080:00200000:2.0:1789762970.411541:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000bad7744a 17ffffc0001001 3:0 ffff936010d4bd20 435 lru 00000080:00200000:2.0:1789762970.411560:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@00000000184fa644 435: 1< 0 - - > 2< 1781760 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411574:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000088e288b 00000080:00200000:2.0:1789762970.411579:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 180. 00000080:00200000:2.0:1789762970.411582:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 436 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411601:0:16269:0:(rw26.c:871:ll_write_end()) pos 1785856, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411627:0:16269:0:(rw26.c:929:ll_write_end()) page@000000007f4540c4[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 436] 00000080:00200000:2.0:1789762970.411652:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @0000000017660f0c 17ffffc0001001 3:0 ffff936010d4b000 436 lru 00000080:00200000:2.0:1789762970.411708:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000007233d10e 436: 1< 0 - - > 2< 1785856 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411724:0:16269:0:(rw26.c:929:ll_write_end()) end page@000000007f4540c4 00000080:00200000:2.0:1789762970.411733:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 181. 00000080:00200000:2.0:1789762970.411738:0:16269:0:(rw26.c:676:ll_write_begin()) Writing 437 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411765:0:16269:0:(rw26.c:871:ll_write_end()) pos 1789952, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411832:0:16269:0:(rw26.c:929:ll_write_end()) page@00000000f13d8829[3 00000000e9cce0bc 2 1 000000001e4c7c38 0 0 437] 00000080:00200000:2.0:1789762970.411852:0:16269:0:(rw26.c:929:ll_write_end()) vmpage @00000000b37650a5 17ffffc0001001 3:0 ffff936010d4b2a0 437 lru 00000080:00200000:2.0:1789762970.411886:0:16269:0:(rw26.c:929:ll_write_end()) osc-page@000000005104b551 437: 1< 0 - - > 2< 1789952 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.411896:0:16269:0:(rw26.c:929:ll_write_end()) end page@00000000f13d8829 00000080:00200000:2.0:1789762970.411902:0:16269:0:(rw26.c:929:ll_write_end()) queued cl_page: 182. 00000080:00200000:2.0:1789762970.411920:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2975 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.411951:0:16279:0:(rw26.c:871:ll_write_end()) pos 12185600, len 4096, copied 4096 00000080:00200000:2.0:1789762970.411982:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000028aed15e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2975] 00000080:00200000:2.0:1789762970.412007:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000745b88cc 17ffffc0001001 3:0 ffff936010d4be00 2975 lru 00000080:00200000:2.0:1789762970.412066:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007c8b3126 1951: 1< 0 - - > 2< 7991296 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412074:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000028aed15e 00000080:00200000:2.0:1789762970.412085:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 160. 00000080:00200000:2.0:1789762970.412094:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2976 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412119:0:16279:0:(rw26.c:871:ll_write_end()) pos 12189696, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412143:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008a8f42e7[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2976] 00000080:00200000:2.0:1789762970.412165:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cd5203bc 17ffffc0001001 3:0 ffff936010d4b1c0 2976 lru 00000080:00200000:2.0:1789762970.412191:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000819bdd45 1952: 1< 0 - - > 2< 7995392 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412205:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008a8f42e7 00000080:00200000:2.0:1789762970.412211:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 161. 00000080:00200000:2.0:1789762970.412221:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2977 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412246:0:16279:0:(rw26.c:871:ll_write_end()) pos 12193792, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412278:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c375504c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2977] 00000080:00200000:2.0:1789762970.412295:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000029a8256f 17ffffc0001001 3:0 ffff936010d4bc40 2977 lru 00000080:00200000:2.0:1789762970.412308:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e406ba46 1953: 1< 0 - - > 2< 7999488 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412311:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c375504c 00000080:00200000:2.0:1789762970.412316:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 162. 00000080:00200000:2.0:1789762970.412339:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2978 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412364:0:16279:0:(rw26.c:871:ll_write_end()) pos 12197888, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412393:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b3c6f0b4[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2978] 00000080:00200000:2.0:1789762970.412416:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000dce116a2 17ffffc0001021 2:0 ffff936010d4ba80 2978 lru 00000080:00200000:2.0:1789762970.412467:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a3931328 1954: 1< 0 - - > 2< 8003584 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412475:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b3c6f0b4 00000080:00200000:2.0:1789762970.412484:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 163. 00000080:00200000:2.0:1789762970.412493:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2979 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412519:0:16279:0:(rw26.c:871:ll_write_end()) pos 12201984, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412525:0:16279:0:(rw26.c:929:ll_write_end()) page@000000004b53b38a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2979] 00000080:00200000:2.0:1789762970.412527:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000002156b7d4 17ffffc0001001 3:0 ffff936010d4b700 2979 lru 00000080:00200000:2.0:1789762970.412532:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000064049053 1955: 1< 0 - - > 2< 8007680 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412542:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000004b53b38a 00000080:00200000:2.0:1789762970.412554:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 164. 00000080:00200000:2.0:1789762970.412562:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2980 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412594:0:16279:0:(rw26.c:871:ll_write_end()) pos 12206080, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412628:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b7768582[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2980] 00000080:00200000:2.0:1789762970.412649:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000072f12e87 17ffffc0001001 3:0 ffff936010ffc700 2980 lru 00000080:00200000:2.0:1789762970.412672:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f97f3175 1956: 1< 0 - - > 2< 8011776 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412687:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b7768582 00000080:00200000:2.0:1789762970.412693:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 165. 00000080:00200000:2.0:1789762970.412696:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2981 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412731:0:16279:0:(rw26.c:871:ll_write_end()) pos 12210176, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412769:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000821e2c19[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2981] 00000080:00200000:2.0:1789762970.412852:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f10550f0 17ffffc0001001 3:0 ffff936010ffc8c0 2981 lru 00000080:00200000:2.0:1789762970.412907:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000730f9ac0 1957: 1< 0 - - > 2< 8015872 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.412919:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000821e2c19 00000080:00200000:2.0:1789762970.412930:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 166. 00000080:00200000:2.0:1789762970.412941:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2982 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.412968:0:16279:0:(rw26.c:871:ll_write_end()) pos 12214272, len 4096, copied 4096 00000080:00200000:2.0:1789762970.412989:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006691d1ff[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2982] 00000080:00200000:2.0:1789762970.412999:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000073ff5ff7 17ffffc0001001 3:0 ffff936010ffc7e0 2982 lru 00000080:00200000:2.0:1789762970.413053:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bae78cde 1958: 1< 0 - - > 2< 8019968 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413069:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006691d1ff 00000080:00200000:2.0:1789762970.413084:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 167. 00000080:00200000:2.0:1789762970.413101:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2983 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413136:0:16279:0:(rw26.c:871:ll_write_end()) pos 12218368, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413142:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008245a76c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2983] 00000080:00200000:2.0:1789762970.413168:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009eaae99f 17ffffc0001001 3:0 ffff936010ffc9a0 2983 lru 00000080:00200000:2.0:1789762970.413233:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003987d4e4 1959: 1< 0 - - > 2< 8024064 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413251:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008245a76c 00000080:00200000:2.0:1789762970.413267:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 168. 00000080:00200000:2.0:1789762970.413276:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2984 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413298:0:16279:0:(rw26.c:871:ll_write_end()) pos 12222464, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413304:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009c047965[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2984] 00000080:00200000:2.0:1789762970.413307:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008afaf529 17ffffc0001001 3:0 ffff936010ffc540 2984 lru 00000080:00200000:2.0:1789762970.413314:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c379f098 1960: 1< 0 - - > 2< 8028160 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413333:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009c047965 00000080:00200000:2.0:1789762970.413342:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 169. 00000080:00200000:2.0:1789762970.413353:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2985 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413384:0:16279:0:(rw26.c:871:ll_write_end()) pos 12226560, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413420:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000042831073[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2985] 00000080:00200000:2.0:1789762970.413435:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c363acaa 17ffffc0001001 3:0 ffff936010ffcb60 2985 lru 00000080:00200000:2.0:1789762970.413476:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000077fe97e5 1961: 1< 0 - - > 2< 8032256 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413485:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000042831073 00000080:00200000:2.0:1789762970.413489:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 170. 00000080:00200000:2.0:1789762970.413498:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2986 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413523:0:16279:0:(rw26.c:871:ll_write_end()) pos 12230656, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413544:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000994250dd[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2986] 00000080:00200000:2.0:1789762970.413565:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001e1de8f1 17ffffc0001001 3:0 ffff936010ffc0e0 2986 lru 00000080:00200000:2.0:1789762970.413619:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000be1abda3 1962: 1< 0 - - > 2< 8036352 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413634:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000994250dd 00000080:00200000:2.0:1789762970.413641:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 171. 00000080:00200000:2.0:1789762970.413647:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2987 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413673:0:16279:0:(rw26.c:871:ll_write_end()) pos 12234752, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413688:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000eaed80a8[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2987] 00000080:00200000:2.0:1789762970.413704:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000c4b7509 17ffffc0001001 3:0 ffff936010ffc620 2987 lru 00000080:00200000:2.0:1789762970.413736:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000867cdb15 1963: 1< 0 - - > 2< 8040448 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413746:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000eaed80a8 00000080:00200000:2.0:1789762970.413749:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 172. 00000080:00200000:2.0:1789762970.413754:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2988 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413779:0:16279:0:(rw26.c:871:ll_write_end()) pos 12238848, len 4096, copied 4096 00000080:00200000:2.0:1789762970.413843:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008dd11168[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2988] 00000080:00200000:2.0:1789762970.413867:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009fce91b8 17ffffc0001001 3:0 ffff936010ffcee0 2988 lru 00000080:00200000:2.0:1789762970.413927:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000500ff804 1964: 1< 0 - - > 2< 8044544 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.413943:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008dd11168 00000080:00200000:2.0:1789762970.413952:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 173. 00000080:00200000:2.0:1789762970.413960:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2989 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.413985:0:16279:0:(rw26.c:871:ll_write_end()) pos 12242944, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414016:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006eb9900e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2989] 00000080:00200000:2.0:1789762970.414039:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d6836a49 17ffffc0001001 3:0 ffff936010ffc460 2989 lru 00000080:00200000:2.0:1789762970.414099:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ebf4a9c9 1965: 1< 0 - - > 2< 8048640 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414112:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006eb9900e 00000080:00200000:2.0:1789762970.414119:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 174. 00000080:00200000:2.0:1789762970.414128:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2990 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414154:0:16279:0:(rw26.c:871:ll_write_end()) pos 12247040, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414159:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000036d98b11[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2990] 00000080:00200000:2.0:1789762970.414161:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000029e50af4 17ffffc0001001 3:0 ffff936010ffc380 2990 lru 00000080:00200000:2.0:1789762970.414166:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000972aa4e3 1966: 1< 0 - - > 2< 8052736 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414169:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000036d98b11 00000080:00200000:2.0:1789762970.414170:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 175. 00000080:00200000:2.0:1789762970.414174:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2991 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414191:0:16279:0:(rw26.c:871:ll_write_end()) pos 12251136, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414224:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000102a0ea3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2991] 00000080:00200000:2.0:1789762970.414252:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000006284502c 17ffffc0001001 3:0 ffff936010ffcd20 2991 lru 00000080:00200000:2.0:1789762970.414300:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000033310f4e 1967: 1< 0 - - > 2< 8056832 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414328:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000102a0ea3 00000080:00200000:2.0:1789762970.414341:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 176. 00000080:00200000:2.0:1789762970.414352:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2992 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414387:0:16279:0:(rw26.c:871:ll_write_end()) pos 12255232, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414423:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000074b0c817[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2992] 00000080:00200000:2.0:1789762970.414434:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ebcd0106 17ffffc0001001 3:0 ffff936010ffc000 2992 lru 00000080:00200000:2.0:1789762970.414494:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b543ef53 1968: 1< 0 - - > 2< 8060928 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414509:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000074b0c817 00000080:00200000:2.0:1789762970.414517:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 177. 00000080:00200000:2.0:1789762970.414526:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2993 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414568:0:16279:0:(rw26.c:871:ll_write_end()) pos 12259328, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414599:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002199731f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2993] 00000080:00200000:2.0:1789762970.414633:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a9ed492c 17ffffc0001021 2:0 ffff936010ffc2a0 2993 lru 00000080:00200000:2.0:1789762970.414707:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000028dbe987 1969: 1< 0 - - > 2< 8065024 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414718:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002199731f 00000080:00200000:2.0:1789762970.414729:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 178. 00000080:00200000:2.0:1789762970.414743:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2994 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414768:0:16279:0:(rw26.c:871:ll_write_end()) pos 12263424, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414818:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002183be7d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2994] 00000080:00200000:2.0:1789762970.414822:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000cb933ae8 17ffffc0001001 3:0 ffff936010ffce00 2994 lru 00000080:00200000:2.0:1789762970.414831:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000cc00e45f 1970: 1< 0 - - > 2< 8069120 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414836:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002183be7d 00000080:00200000:2.0:1789762970.414839:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 179. 00000080:00200000:2.0:1789762970.414847:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2995 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.414888:0:16279:0:(rw26.c:871:ll_write_end()) pos 12267520, len 4096, copied 4096 00000080:00200000:2.0:1789762970.414912:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000da237710[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2995] 00000080:00200000:2.0:1789762970.414927:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000be068ab0 17ffffc0001001 3:0 ffff936010ffc1c0 2995 lru 00000080:00200000:2.0:1789762970.414984:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000080443966 1971: 1< 0 - - > 2< 8073216 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.414998:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000da237710 00000080:00200000:2.0:1789762970.415005:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 180. 00000080:00200000:2.0:1789762970.415016:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2996 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415047:0:16279:0:(rw26.c:871:ll_write_end()) pos 12271616, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415061:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000070823c98[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2996] 00000080:00200000:2.0:1789762970.415076:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008f3444e0 17ffffc0001001 3:0 ffff936010ffcc40 2996 lru 00000080:00200000:2.0:1789762970.415120:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a268a5f7 1972: 1< 0 - - > 2< 8077312 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415142:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000070823c98 00000080:00200000:2.0:1789762970.415151:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 181. 00000080:00200000:2.0:1789762970.415158:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2997 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415179:0:16279:0:(rw26.c:871:ll_write_end()) pos 12275712, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415204:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c20d7c40[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2997] 00000080:00200000:2.0:1789762970.415219:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000071baeb6f 17ffffc0001001 3:0 ffff936010ffca80 2997 lru 00000080:00200000:2.0:1789762970.415247:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ae634833 1973: 1< 0 - - > 2< 8081408 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415252:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c20d7c40 00000080:00200000:2.0:1789762970.415252:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 182. 00000080:00200000:2.0:1789762970.415254:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2998 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415285:0:16279:0:(rw26.c:871:ll_write_end()) pos 12279808, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415305:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c18467e1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2998] 00000080:00200000:2.0:1789762970.415331:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008e67783e 17ffffc0001001 3:0 ffff936004c2cee0 2998 lru 00000080:00200000:2.0:1789762970.415387:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000006a29a338 1974: 1< 0 - - > 2< 8085504 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415411:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c18467e1 00000080:00200000:2.0:1789762970.415418:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 183. 00000080:00200000:2.0:1789762970.415430:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 2999 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415460:0:16279:0:(rw26.c:871:ll_write_end()) pos 12283904, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415487:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000785186f5[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 2999] 00000080:00200000:2.0:1789762970.415511:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000011f1d09b 17ffffc0001001 3:0 ffff936004c2c460 2999 lru 00000080:00200000:2.0:1789762970.415563:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000055bcdd1b 1975: 1< 0 - - > 2< 8089600 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415569:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000785186f5 00000080:00200000:2.0:1789762970.415576:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 184. 00000080:00200000:2.0:1789762970.415589:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3000 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415615:0:16279:0:(rw26.c:871:ll_write_end()) pos 12288000, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415641:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001ce91c89[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3000] 00000080:00200000:2.0:1789762970.415661:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007baa48e3 17ffffc0001001 3:0 ffff936004c2c380 3000 lru 00000080:00200000:2.0:1789762970.415703:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007cad190d 1976: 1< 0 - - > 2< 8093696 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415713:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001ce91c89 00000080:00200000:2.0:1789762970.415724:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 185. 00000080:00200000:2.0:1789762970.415742:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3001 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415780:0:16279:0:(rw26.c:871:ll_write_end()) pos 12292096, len 4096, copied 4096 00000080:00200000:2.0:1789762970.415837:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000da27aee7[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3001] 00000080:00200000:2.0:1789762970.415859:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004ec359cb 17ffffc0001001 3:0 ffff936004c2cd20 3001 lru 00000080:00200000:2.0:1789762970.415926:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000407875d9 1977: 1< 0 - - > 2< 8097792 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.415940:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000da27aee7 00000080:00200000:2.0:1789762970.415949:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 186. 00000080:00200000:2.0:1789762970.415962:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3002 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.415986:0:16279:0:(rw26.c:871:ll_write_end()) pos 12296192, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416000:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e55f2803[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3002] 00000080:00200000:2.0:1789762970.416023:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000094b44da5 17ffffc0001001 3:0 ffff936004c2c000 3002 lru 00000080:00200000:2.0:1789762970.416071:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a3179ea6 1978: 1< 0 - - > 2< 8101888 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416082:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e55f2803 00000080:00200000:2.0:1789762970.416089:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 187. 00000080:00200000:2.0:1789762970.416102:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3003 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416129:0:16279:0:(rw26.c:871:ll_write_end()) pos 12300288, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416154:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000c6d19a54[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3003] 00000080:00200000:2.0:1789762970.416169:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ff00f219 17ffffc0001001 3:0 ffff936004c2c2a0 3003 lru 00000080:00200000:2.0:1789762970.416185:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000069ef56ec 1979: 1< 0 - - > 2< 8105984 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416200:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000c6d19a54 00000080:00200000:2.0:1789762970.416205:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 188. 00000080:00200000:2.0:1789762970.416215:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3004 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416244:0:16279:0:(rw26.c:871:ll_write_end()) pos 12304384, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416278:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000001462460[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3004] 00000080:00200000:2.0:1789762970.416294:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ff00d529 17ffffc0001001 3:0 ffff936004c2ce00 3004 lru 00000080:00200000:2.0:1789762970.416338:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000398ab468 1980: 1< 0 - - > 2< 8110080 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416347:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000001462460 00000080:00200000:2.0:1789762970.416351:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 189. 00000080:00200000:2.0:1789762970.416355:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3005 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416388:0:16279:0:(rw26.c:871:ll_write_end()) pos 12308480, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416423:0:16279:0:(rw26.c:929:ll_write_end()) page@000000000945a3a3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3005] 00000080:00200000:2.0:1789762970.416445:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c0df8fd9 17ffffc0001001 3:0 ffff936004c2c1c0 3005 lru 00000080:00200000:2.0:1789762970.416482:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007afc6ecc 1981: 1< 0 - - > 2< 8114176 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416494:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000000945a3a3 00000080:00200000:2.0:1789762970.416498:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 190. 00000080:00200000:2.0:1789762970.416508:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3006 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416532:0:16279:0:(rw26.c:871:ll_write_end()) pos 12312576, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416556:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001c0df06d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3006] 00000080:00200000:2.0:1789762970.416577:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000743fa086 17ffffc0001001 3:0 ffff936004c2cc40 3006 lru 00000080:00200000:2.0:1789762970.416613:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000474d03e 1982: 1< 0 - - > 2< 8118272 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416626:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001c0df06d 00000080:00200000:2.0:1789762970.416635:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 191. 00000080:00200000:2.0:1789762970.416643:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3007 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416670:0:16279:0:(rw26.c:871:ll_write_end()) pos 12316672, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416688:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000386c972d[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3007] 00000080:00200000:2.0:1789762970.416703:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000375b75bc 17ffffc0001001 3:0 ffff936004c2ca80 3007 lru 00000080:00200000:2.0:1789762970.416723:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000001c5d5246 1983: 1< 0 - - > 2< 8122368 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416726:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000386c972d 00000080:00200000:2.0:1789762970.416727:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 192. 00000080:00200000:2.0:1789762970.416729:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3008 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416754:0:16279:0:(rw26.c:871:ll_write_end()) pos 12320768, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416787:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000050fc4f6[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3008] 00000080:00200000:2.0:1789762970.416840:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000eefdc52f 17ffffc0001021 2:0 ffff936004c2c700 3008 lru 00000080:00200000:2.0:1789762970.416893:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000067abd85e 1984: 1< 0 - - > 2< 8126464 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.416914:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000050fc4f6 00000080:00200000:2.0:1789762970.416923:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 193. 00000080:00200000:2.0:1789762970.416934:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3009 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.416962:0:16279:0:(rw26.c:871:ll_write_end()) pos 12324864, len 4096, copied 4096 00000080:00200000:2.0:1789762970.416986:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d71ee2a1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3009] 00000080:00200000:2.0:1789762970.417010:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004368649f 17ffffc0001001 3:0 ffff936004c2c8c0 3009 lru 00000080:00200000:2.0:1789762970.417066:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ca5f4cc0 1985: 1< 0 - - > 2< 8130560 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417075:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d71ee2a1 00000080:00200000:2.0:1789762970.417080:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 194. 00000080:00200000:2.0:1789762970.417086:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3010 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417103:0:16279:0:(rw26.c:871:ll_write_end()) pos 12328960, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417126:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000091471fce[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3010] 00000080:00200000:2.0:1789762970.417144:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000098072da4 17ffffc0001001 3:0 ffff936004c2c7e0 3010 lru 00000080:00200000:2.0:1789762970.417217:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bb14ae52 1986: 1< 0 - - > 2< 8134656 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417232:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000091471fce 00000080:00200000:2.0:1789762970.417244:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 195. 00000080:00200000:2.0:1789762970.417249:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3011 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417280:0:16279:0:(rw26.c:871:ll_write_end()) pos 12333056, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417295:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d85ee418[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3011] 00000080:00200000:2.0:1789762970.417313:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e7f11dcc 17ffffc0001001 3:0 ffff936004c2c9a0 3011 lru 00000080:00200000:2.0:1789762970.417382:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e5f0e36f 1987: 1< 0 - - > 2< 8138752 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417395:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d85ee418 00000080:00200000:2.0:1789762970.417408:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 196. 00000080:00200000:2.0:1789762970.417422:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3012 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417449:0:16279:0:(rw26.c:871:ll_write_end()) pos 12337152, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417458:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ad27aacc[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3012] 00000080:00200000:2.0:1789762970.417477:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004c1bc933 17ffffc0001001 3:0 ffff936004c2c540 3012 lru 00000080:00200000:2.0:1789762970.417505:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000092cc06e8 1988: 1< 0 - - > 2< 8142848 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417515:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ad27aacc 00000080:00200000:2.0:1789762970.417528:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 197. 00000080:00200000:2.0:1789762970.417539:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3013 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417565:0:16279:0:(rw26.c:871:ll_write_end()) pos 12341248, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417594:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e91f6203[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3013] 00000080:00200000:2.0:1789762970.417622:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000008fc9627b 17ffffc0001001 3:0 ffff936004c2cb60 3013 lru 00000080:00200000:2.0:1789762970.417666:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008b4ede9b 1989: 1< 0 - - > 2< 8146944 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417677:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e91f6203 00000080:00200000:2.0:1789762970.417684:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 198. 00000080:00200000:2.0:1789762970.417690:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3014 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417717:0:16279:0:(rw26.c:871:ll_write_end()) pos 12345344, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417735:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000db9878a1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3014] 00000080:00200000:2.0:1789762970.417757:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d07ef6f1 17ffffc0001001 3:0 ffff936004c2c0e0 3014 lru 00000080:00200000:2.0:1789762970.417821:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000ac068d8c 1990: 1< 0 - - > 2< 8151040 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417825:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000db9878a1 00000080:00200000:2.0:1789762970.417827:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 199. 00000080:00200000:2.0:1789762970.417829:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3015 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417845:0:16279:0:(rw26.c:871:ll_write_end()) pos 12349440, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417850:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000046c2abc0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3015] 00000080:00200000:2.0:1789762970.417854:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000005c17665 17ffffc0001001 3:0 ffff936004c2c620 3015 lru 00000080:00200000:2.0:1789762970.417859:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b9ec0c31 1991: 1< 0 - - > 2< 8155136 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417862:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000046c2abc0 00000080:00200000:2.0:1789762970.417864:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 200. 00000080:00200000:2.0:1789762970.417865:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3016 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417883:0:16279:0:(rw26.c:871:ll_write_end()) pos 12353536, len 4096, copied 4096 00000080:00200000:2.0:1789762970.417914:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000087efca1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3016] 00000080:00200000:2.0:1789762970.417932:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000ab8d9539 17ffffc0001001 3:0 ffff936010ee11c0 3016 lru 00000080:00200000:2.0:1789762970.417949:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008e9f7b7f 1992: 1< 0 - - > 2< 8159232 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.417951:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000087efca1 00000080:00200000:2.0:1789762970.417952:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 201. 00000080:00200000:2.0:1789762970.417955:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3017 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.417977:0:16279:0:(rw26.c:871:ll_write_end()) pos 12357632, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418008:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002f279beb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3017] 00000080:00200000:2.0:1789762970.418037:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000087bfd78 17ffffc0001001 3:0 ffff936010ee1c40 3017 lru 00000080:00200000:2.0:1789762970.418082:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000958b826e 1993: 1< 0 - - > 2< 8163328 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418094:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002f279beb 00000080:00200000:2.0:1789762970.418098:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 202. 00000080:00200000:2.0:1789762970.418105:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3018 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418135:0:16279:0:(rw26.c:871:ll_write_end()) pos 12361728, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418172:0:16279:0:(rw26.c:929:ll_write_end()) page@000000008baa607c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3018] 00000080:00200000:2.0:1789762970.418196:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c243e513 17ffffc0001001 3:0 ffff936010ee1a80 3018 lru 00000080:00200000:2.0:1789762970.418224:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000015bbc2d1 1994: 1< 0 - - > 2< 8167424 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418234:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000008baa607c 00000080:00200000:2.0:1789762970.418249:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 203. 00000080:00200000:2.0:1789762970.418259:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3019 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418283:0:16279:0:(rw26.c:871:ll_write_end()) pos 12365824, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418332:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003a096832[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3019] 00000080:00200000:2.0:1789762970.418348:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004416b0cb 17ffffc0001001 3:0 ffff936010ee1700 3019 lru 00000080:00200000:2.0:1789762970.418391:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000031dfe314 1995: 1< 0 - - > 2< 8171520 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418405:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003a096832 00000080:00200000:2.0:1789762970.418416:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 204. 00000080:00200000:2.0:1789762970.418428:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3020 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418457:0:16279:0:(rw26.c:871:ll_write_end()) pos 12369920, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418473:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000061e6ff8f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3020] 00000080:00200000:2.0:1789762970.418495:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001d30005f 17ffffc0001001 3:0 ffff936010ee18c0 3020 lru 00000080:00200000:2.0:1789762970.418530:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000005a274724 1996: 1< 0 - - > 2< 8175616 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418543:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000061e6ff8f 00000080:00200000:2.0:1789762970.418552:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 205. 00000080:00200000:2.0:1789762970.418563:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3021 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418591:0:16279:0:(rw26.c:871:ll_write_end()) pos 12374016, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418619:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000047b681f3[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3021] 00000080:00200000:2.0:1789762970.418634:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005a36ddd1 17ffffc0001001 3:0 ffff936010ee17e0 3021 lru 00000080:00200000:2.0:1789762970.418660:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008743759e 1997: 1< 0 - - > 2< 8179712 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418664:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000047b681f3 00000080:00200000:2.0:1789762970.418666:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 206. 00000080:00200000:2.0:1789762970.418671:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3022 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418694:0:16279:0:(rw26.c:871:ll_write_end()) pos 12378112, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418711:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000bc663bfb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3022] 00000080:00200000:2.0:1789762970.418728:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000006a073c4 17ffffc0001001 3:0 ffff936010ee19a0 3022 lru 00000080:00200000:2.0:1789762970.418793:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000374ce25e 1998: 1< 0 - - > 2< 8183808 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418837:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000bc663bfb 00000080:00200000:2.0:1789762970.418842:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 207. 00000080:00200000:2.0:1789762970.418847:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3023 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.418873:0:16279:0:(rw26.c:871:ll_write_end()) pos 12382208, len 4096, copied 4096 00000080:00200000:2.0:1789762970.418892:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000fbf5e8a2[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3023] 00000080:00200000:2.0:1789762970.418917:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005821a08e 17ffffc0001021 2:0 ffff936010ee1540 3023 lru 00000080:00200000:2.0:1789762970.418959:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f4c16278 1999: 1< 0 - - > 2< 8187904 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.418968:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000fbf5e8a2 00000080:00200000:2.0:1789762970.418977:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 208. 00000080:00200000:2.0:1789762970.418986:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3024 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419011:0:16279:0:(rw26.c:871:ll_write_end()) pos 12386304, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419037:0:16279:0:(rw26.c:929:ll_write_end()) page@000000009fb77f32[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3024] 00000080:00200000:2.0:1789762970.419059:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000042d3e899 17ffffc0001001 3:0 ffff936010ee1b60 3024 lru 00000080:00200000:2.0:1789762970.419089:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000005f5a42c1 2000: 1< 0 - - > 2< 8192000 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419103:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000009fb77f32 00000080:00200000:2.0:1789762970.419109:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 209. 00000080:00200000:2.0:1789762970.419119:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3025 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419145:0:16279:0:(rw26.c:871:ll_write_end()) pos 12390400, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419177:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000da7782d5[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3025] 00000080:00200000:2.0:1789762970.419195:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003dcd2c42 17ffffc0001001 3:0 ffff936010ee10e0 3025 lru 00000080:00200000:2.0:1789762970.419215:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004cf61e74 2001: 1< 0 - - > 2< 8196096 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419224:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000da7782d5 00000080:00200000:2.0:1789762970.419228:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 210. 00000080:00200000:2.0:1789762970.419239:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3026 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419265:0:16279:0:(rw26.c:871:ll_write_end()) pos 12394496, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419298:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006ff12539[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3026] 00000080:00200000:2.0:1789762970.419329:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003e29fbbd 17ffffc0001001 3:0 ffff936010ee1620 3026 lru 00000080:00200000:2.0:1789762970.419372:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000572c2cf5 2002: 1< 0 - - > 2< 8200192 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419387:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006ff12539 00000080:00200000:2.0:1789762970.419392:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 211. 00000080:00200000:2.0:1789762970.419397:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3027 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419424:0:16279:0:(rw26.c:871:ll_write_end()) pos 12398592, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419456:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000778f3b90[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3027] 00000080:00200000:2.0:1789762970.419472:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007c7257a9 17ffffc0001001 3:0 ffff936010ee1ee0 3027 lru 00000080:00200000:2.0:1789762970.419509:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c43d6cf9 2003: 1< 0 - - > 2< 8204288 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419518:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000778f3b90 00000080:00200000:2.0:1789762970.419524:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 212. 00000080:00200000:2.0:1789762970.419528:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3028 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419549:0:16279:0:(rw26.c:871:ll_write_end()) pos 12402688, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419568:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000215cd130[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3028] 00000080:00200000:2.0:1789762970.419590:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000bab2d2ae 17ffffc0001001 3:0 ffff936010ee1460 3028 lru 00000080:00200000:2.0:1789762970.419628:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004cb6048e 2004: 1< 0 - - > 2< 8208384 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419634:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000215cd130 00000080:00200000:2.0:1789762970.419642:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 213. 00000080:00200000:2.0:1789762970.419649:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3029 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419672:0:16279:0:(rw26.c:871:ll_write_end()) pos 12406784, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419691:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000152ef11f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3029] 00000080:00200000:2.0:1789762970.419696:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000004b44ce99 17ffffc0001001 3:0 ffff936010ee1380 3029 lru 00000080:00200000:2.0:1789762970.419709:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000009b55f6aa 2005: 1< 0 - - > 2< 8212480 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419712:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000152ef11f 00000080:00200000:2.0:1789762970.419714:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 214. 00000080:00200000:2.0:1789762970.419719:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3030 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419743:0:16279:0:(rw26.c:871:ll_write_end()) pos 12410880, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419775:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000f7658f74[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3030] 00000080:00200000:2.0:1789762970.419787:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000015e928ab 17ffffc0001001 3:0 ffff936010ee1d20 3030 lru 00000080:00200000:2.0:1789762970.419867:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000078cc08f8 2006: 1< 0 - - > 2< 8216576 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419884:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000f7658f74 00000080:00200000:2.0:1789762970.419887:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 215. 00000080:00200000:2.0:1789762970.419894:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3031 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.419925:0:16279:0:(rw26.c:871:ll_write_end()) pos 12414976, len 4096, copied 4096 00000080:00200000:2.0:1789762970.419945:0:16279:0:(rw26.c:929:ll_write_end()) page@000000000fa918b7[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3031] 00000080:00200000:2.0:1789762970.419962:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000000b405ad8 17ffffc0001001 3:0 ffff936010ee1000 3031 lru 00000080:00200000:2.0:1789762970.419981:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000075fb6813 2007: 1< 0 - - > 2< 8220672 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.419991:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000000fa918b7 00000080:00200000:2.0:1789762970.420000:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 216. 00000080:00200000:2.0:1789762970.420015:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3032 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420042:0:16279:0:(rw26.c:871:ll_write_end()) pos 12419072, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420073:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a520efd8[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3032] 00000080:00200000:2.0:1789762970.420088:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000004efa048 17ffffc0001001 3:0 ffff936010ee12a0 3032 lru 00000080:00200000:2.0:1789762970.420120:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004dd6c1ac 2008: 1< 0 - - > 2< 8224768 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.420131:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a520efd8 00000080:00200000:2.0:1789762970.420145:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 217. 00000080:00200000:2.0:1789762970.420152:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3033 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420178:0:16279:0:(rw26.c:871:ll_write_end()) pos 12423168, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420211:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ffe59f82[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3033] 00000080:00200000:2.0:1789762970.420232:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009883a5b8 17ffffc0001001 3:0 ffff936010ee1e00 3033 lru 00000080:00200000:2.0:1789762970.420272:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b31998fb 2009: 1< 0 - - > 2< 8228864 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.420279:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ffe59f82 00000080:00200000:2.0:1789762970.420287:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 218. 00000080:00200000:2.0:1789762970.420291:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3034 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420336:0:16279:0:(rw26.c:871:ll_write_end()) pos 12427264, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420360:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000532f2e68[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3034] 00000080:00200000:2.0:1789762970.420379:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000b23f7b24 17ffffc0001001 3:0 ffff936010ee29a0 3034 lru 00000080:00200000:2.0:1789762970.420417:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003118654e 2010: 1< 0 - - > 2< 8232960 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.420430:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000532f2e68 00000080:00200000:2.0:1789762970.420439:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 219. 00000080:00200000:2.0:1789762970.420445:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3035 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420473:0:16279:0:(rw26.c:871:ll_write_end()) pos 12431360, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420503:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b6c9e9d9[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3035] 00000080:00200000:2.0:1789762970.420525:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005fe5e94d 17ffffc0001001 3:0 ffff936010ee2540 3035 lru 00000080:00200000:2.0:1789762970.420561:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a031b788 2011: 1< 0 - - > 2< 8237056 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.420574:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b6c9e9d9 00000080:00200000:2.0:1789762970.420583:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 220. 00000080:00200000:2.0:1789762970.420599:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3036 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420658:0:16279:0:(rw26.c:871:ll_write_end()) pos 12435456, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420672:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006a966e69[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3036] 00000080:00200000:2.0:1789762970.420690:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007f36c647 17ffffc0001001 3:0 ffff936010ee2b60 3036 lru 00000080:00200000:2.0:1789762970.420713:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000bd818197 2012: 1< 0 - - > 2< 8241152 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.420720:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006a966e69 00000080:00200000:2.0:1789762970.420726:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 221. 00000080:00200000:2.0:1789762970.420736:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3037 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420761:0:16279:0:(rw26.c:871:ll_write_end()) pos 12439552, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420781:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000058f2fc1e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3037] 00000080:00200000:2.0:1789762970.420794:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000fe88654f 17ffffc0001001 3:0 ffff936010ee20e0 3037 lru 00000080:00200000:2.0:1789762970.420837:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000057b50081 2013: 1< 0 - - > 2< 8245248 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.420839:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000058f2fc1e 00000080:00200000:2.0:1789762970.420841:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 222. 00000080:00200000:2.0:1789762970.420843:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3038 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420859:0:16279:0:(rw26.c:871:ll_write_end()) pos 12443648, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420865:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000082f6755e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3038] 00000080:00200000:2.0:1789762970.420868:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000009820c365 17ffffc0001021 2:0 ffff936010ee2620 3038 lru 00000080:00200000:2.0:1789762970.420872:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d9968bef 2014: 1< 0 - - > 2< 8249344 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.420874:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000082f6755e 00000080:00200000:2.0:1789762970.420875:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 223. 00000080:00200000:2.0:1789762970.420877:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3039 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.420894:0:16279:0:(rw26.c:871:ll_write_end()) pos 12447744, len 4096, copied 4096 00000080:00200000:2.0:1789762970.420917:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d97ace7f[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3039] 00000080:00200000:2.0:1789762970.420940:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000019978078 17ffffc0001001 3:0 ffff936010ee2ee0 3039 lru 00000080:00200000:2.0:1789762970.420981:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000203687d4 2015: 1< 0 - - > 2< 8253440 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.420992:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d97ace7f 00000080:00200000:2.0:1789762970.420997:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 224. 00000080:00200000:2.0:1789762970.421001:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3040 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421026:0:16279:0:(rw26.c:871:ll_write_end()) pos 12451840, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421036:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000009179945[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3040] 00000080:00200000:2.0:1789762970.421042:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000042a5a212 17ffffc0001001 3:0 ffff936010ee2460 3040 lru 00000080:00200000:2.0:1789762970.421047:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000a51c5afe 2016: 1< 0 - - > 2< 8257536 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 5111808 - | - - + - > 5< - - + - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421051:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000009179945 00000080:00200000:2.0:1789762970.421054:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 225. 00000080:00200000:2.0:1789762970.421061:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3041 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421090:0:16279:0:(rw26.c:871:ll_write_end()) pos 12455936, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421111:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b2210adb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3041] 00000080:00200000:2.0:1789762970.421137:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000001f0e335a 17ffffc0001001 3:0 ffff936010ee2380 3041 lru 00000080:00200000:2.0:1789762970.421160:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b599e1e6 2017: 1< 0 - - > 2< 8261632 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421166:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b2210adb 00000080:00200000:2.0:1789762970.421173:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 226. 00000080:00200000:2.0:1789762970.421176:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3042 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421210:0:16279:0:(rw26.c:871:ll_write_end()) pos 12460032, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421215:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000079e1afbb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3042] 00000080:00200000:2.0:1789762970.421225:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f7948071 17ffffc0001001 3:0 ffff936010ee2d20 3042 lru 00000080:00200000:2.0:1789762970.421252:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004bdb7535 2018: 1< 0 - - > 2< 8265728 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421263:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000079e1afbb 00000080:00200000:2.0:1789762970.421267:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 227. 00000080:00200000:2.0:1789762970.421272:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3043 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421294:0:16279:0:(rw26.c:871:ll_write_end()) pos 12464128, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421329:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000261a3edf[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3043] 00000080:00200000:2.0:1789762970.421338:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e06fab3b 17ffffc0001001 3:0 ffff936010ee2000 3043 lru 00000080:00200000:2.0:1789762970.421367:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000aeb46eda 2019: 1< 0 - - > 2< 8269824 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421372:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000261a3edf 00000080:00200000:2.0:1789762970.421379:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 228. 00000080:00200000:2.0:1789762970.421385:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3044 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421410:0:16279:0:(rw26.c:871:ll_write_end()) pos 12468224, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421424:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000b35c2c94[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3044] 00000080:00200000:2.0:1789762970.421431:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000094c12a56 17ffffc0001001 3:0 ffff936010ee22a0 3044 lru 00000080:00200000:2.0:1789762970.421443:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000f6b677ad 2020: 1< 0 - - > 2< 8273920 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421446:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000b35c2c94 00000080:00200000:2.0:1789762970.421447:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 229. 00000080:00200000:2.0:1789762970.421449:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3045 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421467:0:16279:0:(rw26.c:871:ll_write_end()) pos 12472320, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421486:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ea07d15c[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3045] 00000080:00200000:2.0:1789762970.421506:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000165de7ba 17ffffc0001001 3:0 ffff936010ee2e00 3045 lru 00000080:00200000:2.0:1789762970.421564:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000003a3255cb 2021: 1< 0 - - > 2< 8278016 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421572:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ea07d15c 00000080:00200000:2.0:1789762970.421578:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 230. 00000080:00200000:2.0:1789762970.421586:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3046 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421613:0:16279:0:(rw26.c:871:ll_write_end()) pos 12476416, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421619:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000012e5ffd[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3046] 00000080:00200000:2.0:1789762970.421621:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000077185bbc 17ffffc0001001 3:0 ffff936010ee21c0 3046 lru 00000080:00200000:2.0:1789762970.421625:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000045fe0ecc 2022: 1< 0 - - > 2< 8282112 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421627:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000012e5ffd 00000080:00200000:2.0:1789762970.421628:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 231. 00000080:00200000:2.0:1789762970.421629:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3047 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421646:0:16279:0:(rw26.c:871:ll_write_end()) pos 12480512, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421674:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000ae4cb126[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3047] 00000080:00200000:2.0:1789762970.421686:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000049139b41 17ffffc0001001 3:0 ffff936010ee2c40 3047 lru 00000080:00200000:2.0:1789762970.421727:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000833088da 2023: 1< 0 - - > 2< 8286208 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421735:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000ae4cb126 00000080:00200000:2.0:1789762970.421742:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 232. 00000080:00200000:2.0:1789762970.421751:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3048 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421783:0:16279:0:(rw26.c:871:ll_write_end()) pos 12484608, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421788:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003e4a0ad6[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3048] 00000080:00200000:2.0:1789762970.421792:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007fcec173 17ffffc0001001 3:0 ffff936010ee2a80 3048 lru 00000080:00200000:2.0:1789762970.421825:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000004404a4ec 2024: 1< 0 - - > 2< 8290304 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421828:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003e4a0ad6 00000080:00200000:2.0:1789762970.421832:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 233. 00000080:00200000:2.0:1789762970.421834:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3049 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.421849:0:16279:0:(rw26.c:871:ll_write_end()) pos 12488704, len 4096, copied 4096 00000080:00200000:2.0:1789762970.421873:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002bae0b8e[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3049] 00000080:00200000:2.0:1789762970.421896:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000044e96c17 17ffffc0001001 3:0 ffff936010ee2700 3049 lru 00000080:00200000:2.0:1789762970.421953:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000d142c022 2025: 1< 0 - - > 2< 8294400 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.421961:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002bae0b8e 00000080:00200000:2.0:1789762970.421970:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 234. 00000080:00200000:2.0:1789762970.421980:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3050 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422002:0:16279:0:(rw26.c:871:ll_write_end()) pos 12492800, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422009:0:16279:0:(rw26.c:929:ll_write_end()) page@000000007bda20e1[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3050] 00000080:00200000:2.0:1789762970.422013:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000065f003f1 17ffffc0001001 3:0 ffff936010ee28c0 3050 lru 00000080:00200000:2.0:1789762970.422021:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000056762a54 2026: 1< 0 - - > 2< 8298496 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422026:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000007bda20e1 00000080:00200000:2.0:1789762970.422028:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 235. 00000080:00200000:2.0:1789762970.422032:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3051 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422052:0:16279:0:(rw26.c:871:ll_write_end()) pos 12496896, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422060:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a9882be0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3051] 00000080:00200000:2.0:1789762970.422063:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000e515e5b0 17ffffc0001001 3:0 ffff936010ee27e0 3051 lru 00000080:00200000:2.0:1789762970.422067:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000cee07f9c 2027: 1< 0 - - > 2< 8302592 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422070:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a9882be0 00000080:00200000:2.0:1789762970.422074:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 236. 00000080:00200000:2.0:1789762970.422079:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3052 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422108:0:16279:0:(rw26.c:871:ll_write_end()) pos 12500992, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422118:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a97f7d51[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3052] 00000080:00200000:2.0:1789762970.422127:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000095106768 17ffffc0001001 3:0 ffff936010fa2700 3052 lru 00000080:00200000:2.0:1789762970.422172:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000008153b4b0 2028: 1< 0 - - > 2< 8306688 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422182:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a97f7d51 00000080:00200000:2.0:1789762970.422187:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 237. 00000080:00200000:2.0:1789762970.422191:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3053 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422216:0:16279:0:(rw26.c:871:ll_write_end()) pos 12505088, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422221:0:16279:0:(rw26.c:929:ll_write_end()) page@000000002650bada[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3053] 00000080:00200000:2.0:1789762970.422224:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000007c106cc 17ffffc0001021 2:0 ffff936010fa28c0 3053 lru 00000080:00200000:2.0:1789762970.422237:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000009b14ee64 2029: 1< 0 - - > 2< 8310784 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422241:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000002650bada 00000080:00200000:2.0:1789762970.422243:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 238. 00000080:00200000:2.0:1789762970.422249:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3054 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422272:0:16279:0:(rw26.c:871:ll_write_end()) pos 12509184, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422286:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a6f17583[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3054] 00000080:00200000:2.0:1789762970.422298:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000007b12100b 17ffffc0001001 3:0 ffff936010fa27e0 3054 lru 00000080:00200000:2.0:1789762970.422304:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000c1e3b9e4 2030: 1< 0 - - > 2< 8314880 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422306:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a6f17583 00000080:00200000:2.0:1789762970.422308:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 239. 00000080:00200000:2.0:1789762970.422310:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3055 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422348:0:16279:0:(rw26.c:871:ll_write_end()) pos 12513280, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422364:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000004ae5b66[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3055] 00000080:00200000:2.0:1789762970.422368:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000045d0dd89 17ffffc0001001 3:0 ffff936010fa29a0 3055 lru 00000080:00200000:2.0:1789762970.422374:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000009c45963d 2031: 1< 0 - - > 2< 8318976 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422377:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000004ae5b66 00000080:00200000:2.0:1789762970.422378:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 240. 00000080:00200000:2.0:1789762970.422382:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3056 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422400:0:16279:0:(rw26.c:871:ll_write_end()) pos 12517376, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422407:0:16279:0:(rw26.c:929:ll_write_end()) page@000000007e7ff1c0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3056] 00000080:00200000:2.0:1789762970.422411:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000c20e1e80 17ffffc0001001 3:0 ffff936010fa2540 3056 lru 00000080:00200000:2.0:1789762970.422419:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000cab63afe 2032: 1< 0 - - > 2< 8323072 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422421:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000007e7ff1c0 00000080:00200000:2.0:1789762970.422424:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 241. 00000080:00200000:2.0:1789762970.422427:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3057 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422441:0:16279:0:(rw26.c:871:ll_write_end()) pos 12521472, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422446:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001854f3fa[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3057] 00000080:00200000:2.0:1789762970.422449:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000eb58e038 17ffffc0001001 3:0 ffff936010fa2b60 3057 lru 00000080:00200000:2.0:1789762970.422453:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000594da6d0 2033: 1< 0 - - > 2< 8327168 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422456:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001854f3fa 00000080:00200000:2.0:1789762970.422457:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 242. 00000080:00200000:2.0:1789762970.422459:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3058 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422472:0:16279:0:(rw26.c:871:ll_write_end()) pos 12525568, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422477:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000d9416613[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3058] 00000080:00200000:2.0:1789762970.422480:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f38e6737 17ffffc0001001 3:0 ffff936010fa20e0 3058 lru 00000080:00200000:2.0:1789762970.422484:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000e1e4f077 2034: 1< 0 - - > 2< 8331264 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422486:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000d9416613 00000080:00200000:2.0:1789762970.422488:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 243. 00000080:00200000:2.0:1789762970.422490:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3059 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422503:0:16279:0:(rw26.c:871:ll_write_end()) pos 12529664, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422507:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000086b0168a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3059] 00000080:00200000:2.0:1789762970.422509:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000846d6584 17ffffc0001001 3:0 ffff936010fa2620 3059 lru 00000080:00200000:2.0:1789762970.422514:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000656a36df 2035: 1< 0 - - > 2< 8335360 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422515:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000086b0168a 00000080:00200000:2.0:1789762970.422518:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 244. 00000080:00200000:2.0:1789762970.422520:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3060 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422534:0:16279:0:(rw26.c:871:ll_write_end()) pos 12533760, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422538:0:16279:0:(rw26.c:929:ll_write_end()) page@000000001c5da219[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3060] 00000080:00200000:2.0:1789762970.422540:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000d5901fe0 17ffffc0001001 3:0 ffff936010fa2ee0 3060 lru 00000080:00200000:2.0:1789762970.422544:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000001970be4e 2036: 1< 0 - - > 2< 8339456 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422546:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000001c5da219 00000080:00200000:2.0:1789762970.422547:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 245. 00000080:00200000:2.0:1789762970.422549:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3061 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422565:0:16279:0:(rw26.c:871:ll_write_end()) pos 12537856, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422569:0:16279:0:(rw26.c:929:ll_write_end()) page@000000006a5b6fe0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3061] 00000080:00200000:2.0:1789762970.422571:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000003f69bdbc 17ffffc0001001 3:0 ffff936010fa2460 3061 lru 00000080:00200000:2.0:1789762970.422577:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000587f437d 2037: 1< 0 - - > 2< 8343552 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422579:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000006a5b6fe0 00000080:00200000:2.0:1789762970.422580:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 246. 00000080:00200000:2.0:1789762970.422582:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3062 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422596:0:16279:0:(rw26.c:871:ll_write_end()) pos 12541952, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422601:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a53b3c31[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3062] 00000080:00200000:2.0:1789762970.422604:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f21ebeae 17ffffc0001001 3:0 ffff936010fa2380 3062 lru 00000080:00200000:2.0:1789762970.422609:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000300f63d8 2038: 1< 0 - - > 2< 8347648 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422611:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a53b3c31 00000080:00200000:2.0:1789762970.422613:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 247. 00000080:00200000:2.0:1789762970.422615:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3063 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422627:0:16279:0:(rw26.c:871:ll_write_end()) pos 12546048, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422630:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000000709e7a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3063] 00000080:00200000:2.0:1789762970.422633:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000fc3a991d 17ffffc0001001 3:0 ffff936010fa2d20 3063 lru 00000080:00200000:2.0:1789762970.422637:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000093193d61 2039: 1< 0 - - > 2< 8351744 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422640:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000000709e7a 00000080:00200000:2.0:1789762970.422642:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 248. 00000080:00200000:2.0:1789762970.422644:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3064 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422658:0:16279:0:(rw26.c:871:ll_write_end()) pos 12550144, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422662:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000dfa7b2fb[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3064] 00000080:00200000:2.0:1789762970.422665:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000447d28bc 17ffffc0001001 3:0 ffff936010fa2000 3064 lru 00000080:00200000:2.0:1789762970.422670:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000114e0bc8 2040: 1< 0 - - > 2< 8355840 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422672:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000dfa7b2fb 00000080:00200000:2.0:1789762970.422674:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 249. 00000080:00200000:2.0:1789762970.422676:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3065 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422688:0:16279:0:(rw26.c:871:ll_write_end()) pos 12554240, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422692:0:16279:0:(rw26.c:929:ll_write_end()) page@000000003c2bc337[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3065] 00000080:00200000:2.0:1789762970.422695:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000005ca9c070 17ffffc0001001 3:0 ffff936010fa22a0 3065 lru 00000080:00200000:2.0:1789762970.422700:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000edcf2945 2041: 1< 0 - - > 2< 8359936 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422702:0:16279:0:(rw26.c:929:ll_write_end()) end page@000000003c2bc337 00000080:00200000:2.0:1789762970.422704:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 250. 00000080:00200000:2.0:1789762970.422706:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3066 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422719:0:16279:0:(rw26.c:871:ll_write_end()) pos 12558336, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422723:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000002a62954[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3066] 00000080:00200000:2.0:1789762970.422725:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000006ab278c1 17ffffc0001001 3:0 ffff936010fa2e00 3066 lru 00000080:00200000:2.0:1789762970.422729:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000240ab01e 2042: 1< 0 - - > 2< 8364032 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422732:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000002a62954 00000080:00200000:2.0:1789762970.422734:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 251. 00000080:00200000:2.0:1789762970.422736:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3067 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422748:0:16279:0:(rw26.c:871:ll_write_end()) pos 12562432, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422752:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000a2db2c8a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3067] 00000080:00200000:2.0:1789762970.422755:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @000000006a979126 17ffffc0001001 3:0 ffff936010fa21c0 3067 lru 00000080:00200000:2.0:1789762970.422759:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@00000000b6223fba 2043: 1< 0 - - > 2< 8368128 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422762:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000a2db2c8a 00000080:00200000:2.0:1789762970.422763:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 252. 00000080:00200000:2.0:1789762970.422765:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3068 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422781:0:16279:0:(rw26.c:871:ll_write_end()) pos 12566528, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422785:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000e9c82864[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3068] 00000080:00200000:2.0:1789762970.422787:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000a0976946 17ffffc0001021 2:0 ffff936010fa2c40 3068 lru 00000080:00200000:2.0:1789762970.422793:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000000e73c40f 2044: 1< 0 - - > 2< 8372224 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422795:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000e9c82864 00000080:00200000:2.0:1789762970.422834:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 253. 00000080:00200000:2.0:1789762970.422837:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3069 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422860:0:16279:0:(rw26.c:871:ll_write_end()) pos 12570624, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422866:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000373fb3b0[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3069] 00000080:00200000:2.0:1789762970.422870:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @0000000087098519 17ffffc0001001 3:0 ffff936010fa2a80 3069 lru 00000080:00200000:2.0:1789762970.422877:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@000000007433dc4a 2045: 1< 0 - - > 2< 8376320 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422880:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000373fb3b0 00000080:00200000:2.0:1789762970.422883:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 254. 00000080:00200000:2.0:1789762970.422886:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3070 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422916:0:16279:0:(rw26.c:871:ll_write_end()) pos 12574720, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422922:0:16279:0:(rw26.c:929:ll_write_end()) page@00000000397a687a[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3070] 00000080:00200000:2.0:1789762970.422928:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000f6fa445b 17ffffc0001001 3:0 ffff936004c09380 3070 lru 00000080:00200000:2.0:1789762970.422935:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000019ae18b6 2046: 1< 0 - - > 2< 8380416 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422939:0:16279:0:(rw26.c:929:ll_write_end()) end page@00000000397a687a 00000080:00200000:2.0:1789762970.422941:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 255. 00000080:00200000:2.0:1789762970.422944:0:16279:0:(rw26.c:676:ll_write_begin()) Writing 3071 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.422965:0:16279:0:(rw26.c:871:ll_write_end()) pos 12578816, len 4096, copied 4096 00000080:00200000:2.0:1789762970.422968:0:16279:0:(rw26.c:929:ll_write_end()) page@0000000067c87279[3 00000000e9cce0bc 2 1 00000000ebf61c60 0 0 3071] 00000080:00200000:2.0:1789762970.422974:0:16279:0:(rw26.c:929:ll_write_end()) vmpage @00000000905f076a 17ffffc0001001 3:0 ffff936004c09d20 3071 lru 00000080:00200000:2.0:1789762970.422984:0:16279:0:(rw26.c:929:ll_write_end()) osc-page@0000000028f150eb 2047: 1< 0 - - > 2< 8384512 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 5111808 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.422989:0:16279:0:(rw26.c:929:ll_write_end()) end page@0000000067c87279 00000080:00200000:2.0:1789762970.422992:0:16279:0:(rw26.c:929:ll_write_end()) queued cl_page: 256. 00000080:00200000:2.0:1789762970.422997:0:16279:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:2.0:1789762970.423034:0:16279:0:(osc_cache.c:858:osc_extent_find()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1789762970.423093:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423123:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423155:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423231:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423284:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423362:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423397:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423496:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423529:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423554:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423582:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423616:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423656:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423732:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423789:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423882:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423980:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.423993:0:16279:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 1, dirtied 1 00000080:00200000:2.0:1789762970.424007:0:16279:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:2.0:1789762970.424022:0:16279:0:(vvp_io.c:1358:vvp_io_write_start()) nrs_w_oleg337-client.virtnet: write bytes 0, result: 1048576 00000020:00010000:2.0:1789762970.424051:0:16279:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a4e8 (ffff936008beacf8/0) at cl_lock_release():245 00000020:00010000:2.0:1789762970.424066:0:16279:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a4e8 (ffff936008beacf8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762970.424082:0:16279:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936020ea9ab0 (ffff936008beaea8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1789762970.424104:0:16279:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x810020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.424148:0:16279:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1789762970.424182:0:16279:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a4e8 (ffff936008beacf8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762970.424201:0:16279:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936020ea9ab0 (ffff936008beacf8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762970.424219:0:16279:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:2.0:1789762970.424229:0:16279:0:(file.c:2205:ll_file_io_generic()) Range unlock [2816, 3071] 00000080:00200000:2.0:1789762970.424240:0:16279:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x21:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762970.424260:0:16279:0:(file.c:2238:ll_file_io_generic()) nrs_w_oleg337-client.virtnet: 2 io complete with rc: 0, result: 1048576, restart: 0 00000008:00010000:2.0:1789762970.424290:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 9 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1789762970.424335:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000d568c493 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 9 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1789762970.424371:0:255:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 9 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762970.424852:0:255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (7340032 7344127) ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1789762970.425177:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061df480 - object 0x2c0000401:3 offset 7340032<>8388608 00000008:00010000:2.0:1789762970.425213:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000d568c493 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1789762970.425275:0:255:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000d568c493 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762970.425747:0:255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (5242880 5246975) ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1789762970.426172:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dc700 - object 0x2c0000401:3 offset 5242880<>6291456 00000100:00100000:2.0:1789762970.426234:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067563776:192.168.203.137@tcp:4:dd.0 00000100:00100000:2.0:1789762970.426343:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762970.426358:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067563904:192.168.203.137@tcp:4:dd.0 00000080:00200000:2.0:1789762970.426459:0:16279:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x21:0x0] updating i_size 16777216 i_blocks 2 00000080:00200000:2.0:1789762970.426464:0:16279:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762970.426485:0:16279:0:(file.c:479:ll_file_release()) START file nrs_w_oleg337-client.virtnet:[0x200000402:0x21:0x0](ffff936018e15b88), flags 100002 00000080:00200000:2.0:1789762970.427973:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2732 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.427999:0:16278:0:(rw26.c:871:ll_write_end()) pos 11190272, len 4096, copied 4096 00000080:00200000:2.0:1789762970.428010:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000075f416cf[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2732] 00000080:00200000:2.0:1789762970.428014:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000002459bcfc 17ffffc0001001 3:0 ffff936004c09000 2732 lru 00000080:00200000:2.0:1789762970.428020:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008722c376 1708: 1< 0 - - > 2< 6995968 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.428023:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000075f416cf 00000080:00200000:2.0:1789762970.428024:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 173. 00000080:00200000:2.0:1789762970.428027:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2733 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.428045:0:16278:0:(rw26.c:871:ll_write_end()) pos 11194368, len 4096, copied 4096 00000080:00200000:2.0:1789762970.428066:0:16278:0:(rw26.c:929:ll_write_end()) page@000000004eb7da90[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2733] 00000080:00200000:2.0:1789762970.428080:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000d0643553 17ffffc0001001 3:0 ffff936004c092a0 2733 lru 00000080:00200000:2.0:1789762970.428093:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000aeeb0414 1709: 1< 0 - - > 2< 7000064 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.428099:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000004eb7da90 00000080:00200000:2.0:1789762970.428103:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 174. 00000080:00200000:2.0:1789762970.428108:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2734 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.428129:0:16278:0:(rw26.c:871:ll_write_end()) pos 11198464, len 4096, copied 4096 00000080:00200000:2.0:1789762970.428139:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000c7d64dd0[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2734] 00000080:00200000:2.0:1789762970.428142:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000003dc76e4c 17ffffc0001001 3:0 ffff936004c09e00 2734 lru 00000080:00200000:2.0:1789762970.428147:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000b00651fd 1710: 1< 0 - - > 2< 7004160 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.428149:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000c7d64dd0 00000080:00200000:2.0:1789762970.428151:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 175. 00000080:00200000:2.0:1789762970.428153:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2735 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.428171:0:16278:0:(rw26.c:871:ll_write_end()) pos 11202560, len 4096, copied 4096 00000080:00200000:2.0:1789762970.428180:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b5fc00cc[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2735] 00000080:00200000:2.0:1789762970.428201:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000e4c0512b 17ffffc0001001 3:0 ffff936004c091c0 2735 lru 00000080:00200000:2.0:1789762970.428256:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000bd94ad8 1711: 1< 0 - - > 2< 7008256 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.428261:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b5fc00cc 00000080:00200000:2.0:1789762970.428263:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 176. 00000080:00200000:2.0:1789762970.428269:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2736 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.428285:0:16278:0:(rw26.c:871:ll_write_end()) pos 11206656, len 4096, copied 4096 00000080:00200000:2.0:1789762970.428303:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000042f3b5bc[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2736] 00000080:00200000:2.0:1789762970.428308:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000000eb1ab99 17ffffc0001001 3:0 ffff936004c09c40 2736 lru 00000080:00200000:2.0:1789762970.428333:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000001ac61360 1712: 1< 0 - - > 2< 7012352 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 3407872 - | + - + - > 5< + - + - | 0 - | 256 - -> 00000080:00200000:2.0:1789762970.428338:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000042f3b5bc 00000080:00200000:2.0:1789762970.428342:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 177. 00000080:00200000:2.0:1789762970.428346:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2737 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429273:0:16278:0:(rw26.c:871:ll_write_end()) pos 11210752, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429287:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000d86039f2[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2737] 00000080:00200000:2.0:1789762970.429302:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000002e85d2ca 17ffffc0001001 3:0 ffff936004c09a80 2737 lru 00000080:00200000:2.0:1789762970.429335:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000e91bfda4 1713: 1< 0 - - > 2< 7016448 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429342:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000d86039f2 00000080:00200000:2.0:1789762970.429346:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 178. 00000080:00200000:2.0:1789762970.429371:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2738 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429399:0:16278:0:(rw26.c:871:ll_write_end()) pos 11214848, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429406:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b90dea78[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2738] 00000080:00200000:2.0:1789762970.429410:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000dd4a2295 17ffffc0001001 3:0 ffff936004c09700 2738 lru 00000080:00200000:2.0:1789762970.429418:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000017fa77a8 1714: 1< 0 - - > 2< 7020544 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429421:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b90dea78 00000080:00200000:2.0:1789762970.429423:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 179. 00000080:00200000:2.0:1789762970.429426:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2739 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429443:0:16278:0:(rw26.c:871:ll_write_end()) pos 11218944, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429447:0:16278:0:(rw26.c:929:ll_write_end()) page@000000003d3bab11[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2739] 00000080:00200000:2.0:1789762970.429450:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000003502abf8 17ffffc0001001 3:0 ffff936004c098c0 2739 lru 00000080:00200000:2.0:1789762970.429454:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000fc29f7cf 1715: 1< 0 - - > 2< 7024640 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429456:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000003d3bab11 00000080:00200000:2.0:1789762970.429457:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 180. 00000080:00200000:2.0:1789762970.429459:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2740 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429470:0:16278:0:(rw26.c:871:ll_write_end()) pos 11223040, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429473:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000033d2206f[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2740] 00000080:00200000:2.0:1789762970.429475:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000ecc9e573 17ffffc0001001 3:0 ffff936004c097e0 2740 lru 00000080:00200000:2.0:1789762970.429479:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000f37df922 1716: 1< 0 - - > 2< 7028736 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429481:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000033d2206f 00000080:00200000:2.0:1789762970.429482:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 181. 00000080:00200000:2.0:1789762970.429483:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2741 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429493:0:16278:0:(rw26.c:871:ll_write_end()) pos 11227136, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429496:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000dffb46ba[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2741] 00000080:00200000:2.0:1789762970.429498:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000e9006b53 17ffffc0001001 3:0 ffff936004c099a0 2741 lru 00000080:00200000:2.0:1789762970.429503:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000ce4c4f1 1717: 1< 0 - - > 2< 7032832 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429505:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000dffb46ba 00000080:00200000:2.0:1789762970.429506:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 182. 00000080:00200000:2.0:1789762970.429508:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2742 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429519:0:16278:0:(rw26.c:871:ll_write_end()) pos 11231232, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429523:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000047b50e63[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2742] 00000080:00200000:2.0:1789762970.429525:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f56e5798 17ffffc0001001 3:0 ffff936004c09540 2742 lru 00000080:00200000:2.0:1789762970.429529:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000004ec7b6c7 1718: 1< 0 - - > 2< 7036928 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429531:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000047b50e63 00000080:00200000:2.0:1789762970.429532:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 183. 00000080:00200000:2.0:1789762970.429534:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2743 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429545:0:16278:0:(rw26.c:871:ll_write_end()) pos 11235328, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429549:0:16278:0:(rw26.c:929:ll_write_end()) page@000000004ec6dbaa[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2743] 00000080:00200000:2.0:1789762970.429552:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000093baa181 17ffffc0001001 3:0 ffff936004c09b60 2743 lru 00000080:00200000:2.0:1789762970.429556:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000573a9afe 1719: 1< 0 - - > 2< 7041024 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429559:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000004ec6dbaa 00000080:00200000:2.0:1789762970.429561:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 184. 00000080:00200000:2.0:1789762970.429563:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2744 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429574:0:16278:0:(rw26.c:871:ll_write_end()) pos 11239424, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429579:0:16278:0:(rw26.c:929:ll_write_end()) page@000000007f1c6d87[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2744] 00000080:00200000:2.0:1789762970.429582:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000038e6837e 17ffffc0001001 3:0 ffff936004c090e0 2744 lru 00000080:00200000:2.0:1789762970.429585:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000573d145 1720: 1< 0 - - > 2< 7045120 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429587:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000007f1c6d87 00000080:00200000:2.0:1789762970.429589:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 185. 00000080:00200000:2.0:1789762970.429591:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2745 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429601:0:16278:0:(rw26.c:871:ll_write_end()) pos 11243520, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429605:0:16278:0:(rw26.c:929:ll_write_end()) page@000000005caa4445[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2745] 00000080:00200000:2.0:1789762970.429608:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000002018162 17ffffc0001001 3:0 ffff936004c09620 2745 lru 00000080:00200000:2.0:1789762970.429613:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000ea181e88 1721: 1< 0 - - > 2< 7049216 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429615:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000005caa4445 00000080:00200000:2.0:1789762970.429616:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 186. 00000080:00200000:2.0:1789762970.429618:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2746 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429638:0:16278:0:(rw26.c:871:ll_write_end()) pos 11247616, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429653:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cb14cb45[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2746] 00000080:00200000:2.0:1789762970.429661:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000df95db31 17ffffc0001021 2:0 ffff936004c09ee0 2746 lru 00000080:00200000:2.0:1789762970.429667:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cf366546 1722: 1< 0 - - > 2< 7053312 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429669:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cb14cb45 00000080:00200000:2.0:1789762970.429670:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 187. 00000080:00200000:2.0:1789762970.429671:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2747 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.429685:0:16278:0:(rw26.c:871:ll_write_end()) pos 11251712, len 4096, copied 4096 00000080:00200000:2.0:1789762970.429689:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000ec0fe46d[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2747] 00000080:00200000:2.0:1789762970.429691:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000002c9ca01f 17ffffc0001001 3:0 ffff936004c09460 2747 lru 00000080:00200000:2.0:1789762970.429695:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000005c89aa45 1723: 1< 0 - - > 2< 7057408 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 7 8 3407872 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.429697:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000ec0fe46d 00000080:00200000:2.0:1789762970.429698:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 188. 00000008:00010000:2.0:1789762970.429743:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1789762970.430025:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000004e033074/0xb758ac0edef9cb73 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [7340032->18446744073709551615] (req 7340032->8388607) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aab expref: -99 pid: 16279 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1789762970.430058:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067563776:192.168.203.137@tcp:4:dd.0 00000080:00200000:2.0:1789762970.430077:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2748 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430101:0:16278:0:(rw26.c:871:ll_write_end()) pos 11255808, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430105:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000080c9dcb0[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2748] 00000080:00200000:2.0:1789762970.430108:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000d5c56b3e 17ffffc0001001 3:0 ffff936010e5be00 2748 lru 00000080:00200000:2.0:1789762970.430113:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000f54bf377 1724: 1< 0 - - > 2< 7061504 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430115:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000080c9dcb0 00000080:00200000:2.0:1789762970.430117:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 189. 00000080:00200000:2.0:1789762970.430119:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2749 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430134:0:16278:0:(rw26.c:871:ll_write_end()) pos 11259904, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430138:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b2f1d583[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2749] 00000080:00200000:2.0:1789762970.430140:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008e5e4d4f 17ffffc0001001 3:0 ffff936010e5b1c0 2749 lru 00000080:00200000:2.0:1789762970.430145:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000aaa70a2 1725: 1< 0 - - > 2< 7065600 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430147:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b2f1d583 00000080:00200000:2.0:1789762970.430148:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 190. 00000080:00200000:2.0:1789762970.430150:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2750 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430161:0:16278:0:(rw26.c:871:ll_write_end()) pos 11264000, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430165:0:16278:0:(rw26.c:929:ll_write_end()) page@000000004495d045[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2750] 00000080:00200000:2.0:1789762970.430167:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000006b40444f 17ffffc0001001 3:0 ffff936010e5bc40 2750 lru 00000080:00200000:2.0:1789762970.430171:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cceddb5f 1726: 1< 0 - - > 2< 7069696 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430173:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000004495d045 00000080:00200000:2.0:1789762970.430174:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 191. 00000080:00200000:2.0:1789762970.430176:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2751 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430189:0:16278:0:(rw26.c:871:ll_write_end()) pos 11268096, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430193:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000d65247c4[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2751] 00000080:00200000:2.0:1789762970.430206:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000081b6b0b9 17ffffc0001001 3:0 ffff936010e5ba80 2751 lru 00000080:00200000:2.0:1789762970.430218:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000004f31cab1 1727: 1< 0 - - > 2< 7073792 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430221:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000d65247c4 00000080:00200000:2.0:1789762970.430223:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 192. 00000080:00200000:2.0:1789762970.430224:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2752 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430238:0:16278:0:(rw26.c:871:ll_write_end()) pos 11272192, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430241:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000ed47f325[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2752] 00000080:00200000:2.0:1789762970.430244:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000003312a473 17ffffc0001001 3:0 ffff936010e5b700 2752 lru 00000080:00200000:2.0:1789762970.430248:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000c75ab1b7 1728: 1< 0 - - > 2< 7077888 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430250:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000ed47f325 00000080:00200000:2.0:1789762970.430253:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 193. 00000080:00200000:2.0:1789762970.430259:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2753 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430278:0:16278:0:(rw26.c:871:ll_write_end()) pos 11276288, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430286:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000158a4d3c[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2753] 00000080:00200000:2.0:1789762970.430297:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000ef1cd45c 17ffffc0001001 3:0 ffff936010e5b8c0 2753 lru 00000080:00200000:2.0:1789762970.430302:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000d4b48cb6 1729: 1< 0 - - > 2< 7081984 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430304:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000158a4d3c 00000080:00200000:2.0:1789762970.430306:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 194. 00000080:00200000:2.0:1789762970.430308:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2754 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430340:0:16278:0:(rw26.c:871:ll_write_end()) pos 11280384, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430358:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000fe8deac9[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2754] 00000080:00200000:2.0:1789762970.430365:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000020a0b502 17ffffc0001001 3:0 ffff936010e5b7e0 2754 lru 00000080:00200000:2.0:1789762970.430387:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000bd5c79ad 1730: 1< 0 - - > 2< 7086080 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430395:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000fe8deac9 00000080:00200000:2.0:1789762970.430402:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 195. 00000080:00200000:2.0:1789762970.430410:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2755 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430430:0:16278:0:(rw26.c:871:ll_write_end()) pos 11284480, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430435:0:16278:0:(rw26.c:929:ll_write_end()) page@000000005b30ce1d[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2755] 00000080:00200000:2.0:1789762970.430438:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000af0f0b34 17ffffc0001001 3:0 ffff936010e5b9a0 2755 lru 00000080:00200000:2.0:1789762970.430450:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000d9a9286b 1731: 1< 0 - - > 2< 7090176 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430456:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000005b30ce1d 00000080:00200000:2.0:1789762970.430459:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 196. 00000080:00200000:2.0:1789762970.430463:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2756 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430482:0:16278:0:(rw26.c:871:ll_write_end()) pos 11288576, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430489:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000ebb35354[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2756] 00000080:00200000:2.0:1789762970.430500:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008825b6a7 17ffffc0001001 3:0 ffff936010e5b540 2756 lru 00000080:00200000:2.0:1789762970.430512:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000785d4e17 1732: 1< 0 - - > 2< 7094272 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430515:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000ebb35354 00000080:00200000:2.0:1789762970.430516:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 197. 00000080:00200000:2.0:1789762970.430518:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2757 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430533:0:16278:0:(rw26.c:871:ll_write_end()) pos 11292672, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430551:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000f077a702[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2757] 00000080:00200000:2.0:1789762970.430553:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000019aabf36 17ffffc0001001 3:0 ffff936010e5bb60 2757 lru 00000080:00200000:2.0:1789762970.430557:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000dbde9e47 1733: 1< 0 - - > 2< 7098368 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430559:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000f077a702 00000080:00200000:2.0:1789762970.430560:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 198. 00000080:00200000:2.0:1789762970.430561:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2758 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430571:0:16278:0:(rw26.c:871:ll_write_end()) pos 11296768, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430576:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cb9ffaef[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2758] 00000080:00200000:2.0:1789762970.430579:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f4b15722 17ffffc0001001 3:0 ffff936010e5b0e0 2758 lru 00000080:00200000:2.0:1789762970.430585:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000a175b845 1734: 1< 0 - - > 2< 7102464 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430587:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cb9ffaef 00000080:00200000:2.0:1789762970.430588:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 199. 00000080:00200000:2.0:1789762970.430591:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2759 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430606:0:16278:0:(rw26.c:871:ll_write_end()) pos 11300864, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430611:0:16278:0:(rw26.c:929:ll_write_end()) page@000000000a6a4169[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2759] 00000080:00200000:2.0:1789762970.430614:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000058636a2d 17ffffc0001001 3:0 ffff936010e5b620 2759 lru 00000080:00200000:2.0:1789762970.430620:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000061ea458a 1735: 1< 0 - - > 2< 7106560 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430623:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000000a6a4169 00000080:00200000:2.0:1789762970.430625:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 200. 00000080:00200000:2.0:1789762970.430627:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2760 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430643:0:16278:0:(rw26.c:871:ll_write_end()) pos 11304960, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430648:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000012340050[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2760] 00000080:00200000:2.0:1789762970.430651:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008fb8e0c2 17ffffc0001001 3:0 ffff936010e5bee0 2760 lru 00000080:00200000:2.0:1789762970.430657:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000002781e4c1 1736: 1< 0 - - > 2< 7110656 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430659:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000012340050 00000080:00200000:2.0:1789762970.430660:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 201. 00000080:00200000:2.0:1789762970.430662:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2761 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430681:0:16278:0:(rw26.c:871:ll_write_end()) pos 11309056, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430686:0:16278:0:(rw26.c:929:ll_write_end()) page@000000003dfad5a8[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2761] 00000080:00200000:2.0:1789762970.430689:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000aac9d6e5 17ffffc0001021 2:0 ffff936010e5b460 2761 lru 00000080:00200000:2.0:1789762970.430695:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008865b41a 1737: 1< 0 - - > 2< 7114752 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430697:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000003dfad5a8 00000080:00200000:2.0:1789762970.430699:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 202. 00000080:00200000:2.0:1789762970.430700:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2762 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430716:0:16278:0:(rw26.c:871:ll_write_end()) pos 11313152, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430722:0:16278:0:(rw26.c:929:ll_write_end()) page@000000006e11e96d[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2762] 00000080:00200000:2.0:1789762970.430726:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000e61d3f1b 17ffffc0001001 3:0 ffff936010e5b380 2762 lru 00000080:00200000:2.0:1789762970.430733:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000b77a2d6a 1738: 1< 0 - - > 2< 7118848 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430735:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000006e11e96d 00000080:00200000:2.0:1789762970.430737:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 203. 00000080:00200000:2.0:1789762970.430739:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2763 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430754:0:16278:0:(rw26.c:871:ll_write_end()) pos 11317248, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430760:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000d11e8772[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2763] 00000080:00200000:2.0:1789762970.430763:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000010d3d8ff 17ffffc0001001 3:0 ffff936010e5bd20 2763 lru 00000080:00200000:2.0:1789762970.430770:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000017e707c7 1739: 1< 0 - - > 2< 7122944 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430773:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000d11e8772 00000080:00200000:2.0:1789762970.430774:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 204. 00000080:00200000:2.0:1789762970.430777:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2764 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430793:0:16278:0:(rw26.c:871:ll_write_end()) pos 11321344, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430826:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000883ba667[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2764] 00000080:00200000:2.0:1789762970.430831:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000bf30baba 17ffffc0001001 3:0 ffff936010e5b000 2764 lru 00000080:00200000:2.0:1789762970.430837:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000001d45257f 1740: 1< 0 - - > 2< 7127040 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430841:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000883ba667 00000080:00200000:2.0:1789762970.430844:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 205. 00000080:00200000:2.0:1789762970.430847:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2765 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430867:0:16278:0:(rw26.c:871:ll_write_end()) pos 11325440, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430873:0:16278:0:(rw26.c:929:ll_write_end()) page@000000006a80c4cc[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2765] 00000080:00200000:2.0:1789762970.430877:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000cb183a11 17ffffc0001001 3:0 ffff936010e5b2a0 2765 lru 00000080:00200000:2.0:1789762970.430883:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000001704a01d 1741: 1< 0 - - > 2< 7131136 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430887:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000006a80c4cc 00000080:00200000:2.0:1789762970.430889:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 206. 00000080:00200000:2.0:1789762970.430917:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2766 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430941:0:16278:0:(rw26.c:871:ll_write_end()) pos 11329536, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430954:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000690710ae[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2766] 00000080:00200000:2.0:1789762970.430963:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000005d438535 17ffffc0001001 3:0 ffff936010f5d1c0 2766 lru 00000080:00200000:2.0:1789762970.430969:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000096b67694 1742: 1< 0 - - > 2< 7135232 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.430973:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000690710ae 00000080:00200000:2.0:1789762970.430974:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 207. 00000080:00200000:2.0:1789762970.430977:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2767 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.430994:0:16278:0:(rw26.c:871:ll_write_end()) pos 11333632, len 4096, copied 4096 00000080:00200000:2.0:1789762970.430998:0:16278:0:(rw26.c:929:ll_write_end()) page@000000009a14c309[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2767] 00000080:00200000:2.0:1789762970.431002:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000e0255951 17ffffc0001001 3:0 ffff936010f5dc40 2767 lru 00000080:00200000:2.0:1789762970.431008:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cf454a99 1743: 1< 0 - - > 2< 7139328 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431011:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000009a14c309 00000080:00200000:2.0:1789762970.431012:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 208. 00000080:00200000:2.0:1789762970.431014:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2768 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431027:0:16278:0:(rw26.c:871:ll_write_end()) pos 11337728, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431031:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000c7fa99ba[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2768] 00000080:00200000:2.0:1789762970.431035:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000070fe4d82 17ffffc0001001 3:0 ffff936010f5da80 2768 lru 00000080:00200000:2.0:1789762970.431040:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008ab7b6ca 1744: 1< 0 - - > 2< 7143424 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431043:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000c7fa99ba 00000080:00200000:2.0:1789762970.431045:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 209. 00000080:00200000:2.0:1789762970.431047:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2769 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431062:0:16278:0:(rw26.c:871:ll_write_end()) pos 11341824, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431066:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000fbcc9738[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2769] 00000080:00200000:2.0:1789762970.431069:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000000f76595e 17ffffc0001001 3:0 ffff936010f5d700 2769 lru 00000080:00200000:2.0:1789762970.431075:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000025dbf8ba 1745: 1< 0 - - > 2< 7147520 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431077:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000fbcc9738 00000080:00200000:2.0:1789762970.431079:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 210. 00000080:00200000:2.0:1789762970.431081:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2770 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431099:0:16278:0:(rw26.c:871:ll_write_end()) pos 11345920, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431105:0:16278:0:(rw26.c:929:ll_write_end()) page@000000002359f06e[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2770] 00000080:00200000:2.0:1789762970.431109:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f8cacddb 17ffffc0001001 3:0 ffff936010f5d8c0 2770 lru 00000080:00200000:2.0:1789762970.431115:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000010803aa1 1746: 1< 0 - - > 2< 7151616 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431118:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000002359f06e 00000080:00200000:2.0:1789762970.431120:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 211. 00000080:00200000:2.0:1789762970.431123:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2771 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431140:0:16278:0:(rw26.c:871:ll_write_end()) pos 11350016, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431146:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000c51d0d5c[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2771] 00000080:00200000:2.0:1789762970.431150:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000086faee35 17ffffc0001001 3:0 ffff936010f5d7e0 2771 lru 00000080:00200000:2.0:1789762970.431157:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000b92dd9a4 1747: 1< 0 - - > 2< 7155712 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431160:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000c51d0d5c 00000080:00200000:2.0:1789762970.431162:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 212. 00000080:00200000:2.0:1789762970.431164:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2772 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431179:0:16278:0:(rw26.c:871:ll_write_end()) pos 11354112, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431183:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000004def47f[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2772] 00000080:00200000:2.0:1789762970.431185:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000044cfbb56 17ffffc0001001 3:0 ffff936010f5d9a0 2772 lru 00000080:00200000:2.0:1789762970.431190:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008c0cb764 1748: 1< 0 - - > 2< 7159808 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431193:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000004def47f 00000080:00200000:2.0:1789762970.431194:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 213. 00000080:00200000:2.0:1789762970.431195:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2773 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431204:0:16278:0:(rw26.c:871:ll_write_end()) pos 11358208, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431208:0:16278:0:(rw26.c:929:ll_write_end()) page@000000002b003703[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2773] 00000080:00200000:2.0:1789762970.431210:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008bff6f37 17ffffc0001001 3:0 ffff936010f5d540 2773 lru 00000080:00200000:2.0:1789762970.431214:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000d577578 1749: 1< 0 - - > 2< 7163904 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431216:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000002b003703 00000080:00200000:2.0:1789762970.431217:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 214. 00000080:00200000:2.0:1789762970.431218:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2774 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431233:0:16278:0:(rw26.c:871:ll_write_end()) pos 11362304, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431255:0:16278:0:(rw26.c:929:ll_write_end()) page@000000001fd28235[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2774] 00000080:00200000:2.0:1789762970.431262:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000001e09b0cb 17ffffc0001001 3:0 ffff936010f5db60 2774 lru 00000080:00200000:2.0:1789762970.431274:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000d7031677 1750: 1< 0 - - > 2< 7168000 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431278:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000001fd28235 00000080:00200000:2.0:1789762970.431282:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 215. 00000080:00200000:2.0:1789762970.431286:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2775 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431297:0:16278:0:(rw26.c:871:ll_write_end()) pos 11366400, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431304:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000064161605[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2775] 00000080:00200000:2.0:1789762970.431309:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f89aff23 17ffffc0001001 3:0 ffff936010f5d0e0 2775 lru 00000080:00200000:2.0:1789762970.431334:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000857a8262 1751: 1< 0 - - > 2< 7172096 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431337:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000064161605 00000080:00200000:2.0:1789762970.431338:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 216. 00000080:00200000:2.0:1789762970.431340:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2776 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431357:0:16278:0:(rw26.c:871:ll_write_end()) pos 11370496, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431360:0:16278:0:(rw26.c:929:ll_write_end()) page@000000002d74179b[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2776] 00000080:00200000:2.0:1789762970.431362:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000229f225c 17ffffc0001021 2:0 ffff936010f5d620 2776 lru 00000080:00200000:2.0:1789762970.431366:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000007937cce1 1752: 1< 0 - - > 2< 7176192 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431368:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000002d74179b 00000080:00200000:2.0:1789762970.431369:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 217. 00000080:00200000:2.0:1789762970.431371:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2777 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431386:0:16278:0:(rw26.c:871:ll_write_end()) pos 11374592, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431391:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cf458ecb[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2777] 00000080:00200000:2.0:1789762970.431395:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000000e43da55 17ffffc0001001 3:0 ffff936010f5dee0 2777 lru 00000080:00200000:2.0:1789762970.431401:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000ce0660f9 1753: 1< 0 - - > 2< 7180288 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431404:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cf458ecb 00000080:00200000:2.0:1789762970.431405:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 218. 00000080:00200000:2.0:1789762970.431408:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2778 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431422:0:16278:0:(rw26.c:871:ll_write_end()) pos 11378688, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431427:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000089ddbfed[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2778] 00000080:00200000:2.0:1789762970.431429:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008b30f950 17ffffc0001001 3:0 ffff936010f5d460 2778 lru 00000080:00200000:2.0:1789762970.431433:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000081305e39 1754: 1< 0 - - > 2< 7184384 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431435:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000089ddbfed 00000080:00200000:2.0:1789762970.431436:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 219. 00000080:00200000:2.0:1789762970.431437:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2779 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431447:0:16278:0:(rw26.c:871:ll_write_end()) pos 11382784, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431450:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000bf62ae76[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2779] 00000080:00200000:2.0:1789762970.431451:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000045822b3c 17ffffc0001001 3:0 ffff936010f5d380 2779 lru 00000080:00200000:2.0:1789762970.431455:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000006ec4dcd0 1755: 1< 0 - - > 2< 7188480 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431457:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000bf62ae76 00000080:00200000:2.0:1789762970.431460:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 220. 00000080:00200000:2.0:1789762970.431461:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2780 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431470:0:16278:0:(rw26.c:871:ll_write_end()) pos 11386880, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431473:0:16278:0:(rw26.c:929:ll_write_end()) page@000000003ad5ca09[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2780] 00000080:00200000:2.0:1789762970.431476:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000414ceddd 17ffffc0001001 3:0 ffff936010f5dd20 2780 lru 00000080:00200000:2.0:1789762970.431480:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000e26fc8c9 1756: 1< 0 - - > 2< 7192576 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431482:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000003ad5ca09 00000080:00200000:2.0:1789762970.431483:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 221. 00000080:00200000:2.0:1789762970.431484:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2781 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431492:0:16278:0:(rw26.c:871:ll_write_end()) pos 11390976, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431495:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000da9ba8fe[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2781] 00000080:00200000:2.0:1789762970.431497:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000001c7fc0b0 17ffffc0001001 3:0 ffff936010f5d000 2781 lru 00000080:00200000:2.0:1789762970.431500:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cd14c303 1757: 1< 0 - - > 2< 7196672 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431501:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000da9ba8fe 00000080:00200000:2.0:1789762970.431504:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 222. 00000080:00200000:2.0:1789762970.431505:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2782 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431513:0:16278:0:(rw26.c:871:ll_write_end()) pos 11395072, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431516:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b8678386[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2782] 00000080:00200000:2.0:1789762970.431519:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000000752e18d 17ffffc0001001 3:0 ffff936010f5d2a0 2782 lru 00000080:00200000:2.0:1789762970.431523:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000076cc0899 1758: 1< 0 - - > 2< 7200768 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431525:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b8678386 00000080:00200000:2.0:1789762970.431527:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 223. 00000080:00200000:2.0:1789762970.431528:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2783 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431537:0:16278:0:(rw26.c:871:ll_write_end()) pos 11399168, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431539:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000e9874c31[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2783] 00000080:00200000:2.0:1789762970.431541:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000000c113de1 17ffffc0001001 3:0 ffff936010f5de00 2783 lru 00000080:00200000:2.0:1789762970.431545:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000015e955e4 1759: 1< 0 - - > 2< 7204864 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431547:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000e9874c31 00000080:00200000:2.0:1789762970.431548:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 224. 00000080:00200000:2.0:1789762970.431550:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2784 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431562:0:16278:0:(rw26.c:871:ll_write_end()) pos 11403264, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431566:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000005620ef[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2784] 00000080:00200000:2.0:1789762970.431569:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000004df0c51c 17ffffc0001001 3:0 ffff936004c140e0 2784 lru 00000080:00200000:2.0:1789762970.431573:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000001cfc9283 1760: 1< 0 - - > 2< 7208960 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431575:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000005620ef 00000080:00200000:2.0:1789762970.431576:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 225. 00000080:00200000:2.0:1789762970.431578:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2785 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431588:0:16278:0:(rw26.c:871:ll_write_end()) pos 11407360, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431592:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000487b634a[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2785] 00000080:00200000:2.0:1789762970.431594:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000fe799259 17ffffc0001001 3:0 ffff936004c14620 2785 lru 00000080:00200000:2.0:1789762970.431598:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000a77fc2a2 1761: 1< 0 - - > 2< 7213056 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431600:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000487b634a 00000080:00200000:2.0:1789762970.431601:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 226. 00000080:00200000:2.0:1789762970.431603:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2786 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431614:0:16278:0:(rw26.c:871:ll_write_end()) pos 11411456, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431619:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000a0b17cdb[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2786] 00000080:00200000:2.0:1789762970.431626:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000030e7fa66 17ffffc0001001 3:0 ffff936004c14ee0 2786 lru 00000080:00200000:2.0:1789762970.431647:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000001c4dae80 1762: 1< 0 - - > 2< 7217152 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 6 8 6815744 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431651:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000a0b17cdb 00000080:00200000:2.0:1789762970.431652:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 227. 00000080:00200000:2.0:1789762970.431654:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2787 of 0 to 4096 bytes 00000008:00010000:2.0:1789762970.431696:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000d568c493 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1789762970.431920:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000d568c493 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000000da80434/0xb758ac0edef9cb65 lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [5242880->6291455] (req 5242880->6291455) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a9d expref: -99 pid: 16277 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1789762970.431943:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067563904:192.168.203.137@tcp:4:dd.0 00000080:00200000:2.0:1789762970.431965:0:16278:0:(rw26.c:871:ll_write_end()) pos 11415552, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431969:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000052ebacac[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2787] 00000080:00200000:2.0:1789762970.431972:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000a18b798e 17ffffc0001001 3:0 ffff936004c14460 2787 lru 00000080:00200000:2.0:1789762970.431977:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000077598c92 1763: 1< 0 - - > 2< 7221248 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.431979:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000052ebacac 00000080:00200000:2.0:1789762970.431981:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 228. 00000080:00200000:2.0:1789762970.431983:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2788 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.431995:0:16278:0:(rw26.c:871:ll_write_end()) pos 11419648, len 4096, copied 4096 00000080:00200000:2.0:1789762970.431998:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000012a4cb8c[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2788] 00000080:00200000:2.0:1789762970.432001:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000092c21108 17ffffc0001001 3:0 ffff936004c14380 2788 lru 00000080:00200000:2.0:1789762970.432005:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cea16463 1764: 1< 0 - - > 2< 7225344 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432007:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000012a4cb8c 00000080:00200000:2.0:1789762970.432008:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 229. 00000080:00200000:2.0:1789762970.432009:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2789 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432026:0:16278:0:(rw26.c:871:ll_write_end()) pos 11423744, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432029:0:16278:0:(rw26.c:929:ll_write_end()) page@000000009a8a0e05[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2789] 00000080:00200000:2.0:1789762970.432037:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000004647515 17ffffc0001001 3:0 ffff936004c14d20 2789 lru 00000080:00200000:2.0:1789762970.432055:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000fdec4740 1765: 1< 0 - - > 2< 7229440 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432060:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000009a8a0e05 00000080:00200000:2.0:1789762970.432063:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 230. 00000080:00200000:2.0:1789762970.432071:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2790 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432087:0:16278:0:(rw26.c:871:ll_write_end()) pos 11427840, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432094:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000013a0ae6b[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2790] 00000080:00200000:2.0:1789762970.432103:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000004c01b572 17ffffc0001001 3:0 ffff936004c14000 2790 lru 00000080:00200000:2.0:1789762970.432113:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000003be14ef1 1766: 1< 0 - - > 2< 7233536 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432120:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000013a0ae6b 00000080:00200000:2.0:1789762970.432121:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 231. 00000080:00200000:2.0:1789762970.432123:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2791 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432142:0:16278:0:(rw26.c:871:ll_write_end()) pos 11431936, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432170:0:16278:0:(rw26.c:929:ll_write_end()) page@000000002b0275f1[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2791] 00000080:00200000:2.0:1789762970.432176:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000009187c150 17ffffc0001021 2:0 ffff936004c142a0 2791 lru 00000080:00200000:2.0:1789762970.432189:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000eaf806ac 1767: 1< 0 - - > 2< 7237632 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432196:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000002b0275f1 00000080:00200000:2.0:1789762970.432200:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 232. 00000080:00200000:2.0:1789762970.432205:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2792 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432221:0:16278:0:(rw26.c:871:ll_write_end()) pos 11436032, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432231:0:16278:0:(rw26.c:929:ll_write_end()) page@000000005b0724da[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2792] 00000080:00200000:2.0:1789762970.432235:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000007d60128a 17ffffc0001001 3:0 ffff936004c14e00 2792 lru 00000080:00200000:2.0:1789762970.432239:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000892ddb81 1768: 1< 0 - - > 2< 7241728 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432240:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000005b0724da 00000080:00200000:2.0:1789762970.432241:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 233. 00000080:00200000:2.0:1789762970.432242:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2793 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432257:0:16278:0:(rw26.c:871:ll_write_end()) pos 11440128, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432268:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000bf63c647[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2793] 00000080:00200000:2.0:1789762970.432278:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f6d93f12 17ffffc0001001 3:0 ffff936004c141c0 2793 lru 00000080:00200000:2.0:1789762970.432288:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000be5b6096 1769: 1< 0 - - > 2< 7245824 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432294:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000bf63c647 00000080:00200000:2.0:1789762970.432297:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 234. 00000080:00200000:2.0:1789762970.432304:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2794 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432335:0:16278:0:(rw26.c:871:ll_write_end()) pos 11444224, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432347:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000093e44669[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2794] 00000080:00200000:2.0:1789762970.432351:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000415d8ae2 17ffffc0001001 3:0 ffff936004c14c40 2794 lru 00000080:00200000:2.0:1789762970.432362:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000ced633bd 1770: 1< 0 - - > 2< 7249920 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432365:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000093e44669 00000080:00200000:2.0:1789762970.432367:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 235. 00000080:00200000:2.0:1789762970.432368:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2795 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432386:0:16278:0:(rw26.c:871:ll_write_end()) pos 11448320, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432390:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000bc34de1c[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2795] 00000080:00200000:2.0:1789762970.432393:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000363648e5 17ffffc0001001 3:0 ffff936004c14a80 2795 lru 00000080:00200000:2.0:1789762970.432400:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000ef0d167c 1771: 1< 0 - - > 2< 7254016 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432403:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000bc34de1c 00000080:00200000:2.0:1789762970.432405:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 236. 00000080:00200000:2.0:1789762970.432407:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2796 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432424:0:16278:0:(rw26.c:871:ll_write_end()) pos 11452416, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432429:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cbf37107[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2796] 00000080:00200000:2.0:1789762970.432432:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000001fbf3b0c 17ffffc0001001 3:0 ffff936004c14700 2796 lru 00000080:00200000:2.0:1789762970.432436:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000022dfaefd 1772: 1< 0 - - > 2< 7258112 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432438:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cbf37107 00000080:00200000:2.0:1789762970.432439:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 237. 00000080:00200000:2.0:1789762970.432441:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2797 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432452:0:16278:0:(rw26.c:871:ll_write_end()) pos 11456512, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432455:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000801d10a2[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2797] 00000080:00200000:2.0:1789762970.432458:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000d8de1c59 17ffffc0001001 3:0 ffff936004c148c0 2797 lru 00000080:00200000:2.0:1789762970.432462:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000006332942a 1773: 1< 0 - - > 2< 7262208 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432464:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000801d10a2 00000080:00200000:2.0:1789762970.432466:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 238. 00000080:00200000:2.0:1789762970.432473:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2798 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432490:0:16278:0:(rw26.c:871:ll_write_end()) pos 11460608, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432493:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b6d0d370[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2798] 00000080:00200000:2.0:1789762970.432495:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000084ec1e17 17ffffc0001001 3:0 ffff936004c147e0 2798 lru 00000080:00200000:2.0:1789762970.432500:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000cb016323 1774: 1< 0 - - > 2< 7266304 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 10223616 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432502:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b6d0d370 00000080:00200000:2.0:1789762970.432503:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 239. 00000080:00200000:2.0:1789762970.432505:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2799 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432514:0:16278:0:(rw26.c:871:ll_write_end()) pos 11464704, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432533:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000060aed408[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2799] 00000080:00200000:2.0:1789762970.432541:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000002f765f2f 17ffffc0001001 3:0 ffff936004c149a0 2799 lru 00000080:00200000:2.0:1789762970.432557:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000f6c44c87 1775: 1< 0 - - > 2< 7270400 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432563:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000060aed408 00000080:00200000:2.0:1789762970.432567:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 240. 00000080:00200000:2.0:1789762970.432573:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2800 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432590:0:16278:0:(rw26.c:871:ll_write_end()) pos 11468800, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432606:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000efb9fdbc[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2800] 00000080:00200000:2.0:1789762970.432617:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000c08fb838 17ffffc0001001 3:0 ffff936004c14540 2800 lru 00000080:00200000:2.0:1789762970.432628:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000051fe1a55 1776: 1< 0 - - > 2< 7274496 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432635:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000efb9fdbc 00000080:00200000:2.0:1789762970.432636:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 241. 00000080:00200000:2.0:1789762970.432638:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2801 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432650:0:16278:0:(rw26.c:871:ll_write_end()) pos 11472896, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432653:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000571e9803[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2801] 00000080:00200000:2.0:1789762970.432656:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000001ec16c70 17ffffc0001001 3:0 ffff936004c14b60 2801 lru 00000080:00200000:2.0:1789762970.432660:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000842bb8f2 1777: 1< 0 - - > 2< 7278592 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432661:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000571e9803 00000080:00200000:2.0:1789762970.432662:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 242. 00000080:00200000:2.0:1789762970.432665:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2802 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432702:0:16278:0:(rw26.c:871:ll_write_end()) pos 11476992, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432712:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000da8be5d1[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2802] 00000080:00200000:2.0:1789762970.432718:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000061d17193 17ffffc0001001 3:0 ffff9360110dc380 2802 lru 00000080:00200000:2.0:1789762970.432728:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000002ecceb53 1778: 1< 0 - - > 2< 7282688 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432734:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000da8be5d1 00000080:00200000:2.0:1789762970.432735:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 243. 00000080:00200000:2.0:1789762970.432736:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2803 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432759:0:16278:0:(rw26.c:871:ll_write_end()) pos 11481088, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432768:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000058e9b6d5[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2803] 00000080:00200000:2.0:1789762970.432780:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000fcb6a693 17ffffc0001001 3:0 ffff9360110dcd20 2803 lru 00000080:00200000:2.0:1789762970.432843:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000e8b10562 1779: 1< 0 - - > 2< 7286784 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 5 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432851:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000058e9b6d5 00000080:00200000:2.0:1789762970.432857:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 244. 00000080:00200000:2.0:1789762970.432864:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2804 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432883:0:16278:0:(rw26.c:871:ll_write_end()) pos 11485184, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432891:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000b4a90b30[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2804] 00000080:00200000:2.0:1789762970.432907:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000f396be93 17ffffc0001001 3:0 ffff9360110dc000 2804 lru 00000080:00200000:2.0:1789762970.432916:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000e1a0b92c 1780: 1< 0 - - > 2< 7290880 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432920:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000b4a90b30 00000080:00200000:2.0:1789762970.432921:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 245. 00000080:00200000:2.0:1789762970.432925:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2805 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.432949:0:16278:0:(rw26.c:871:ll_write_end()) pos 11489280, len 4096, copied 4096 00000080:00200000:2.0:1789762970.432963:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000048cd8bb3[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2805] 00000080:00200000:2.0:1789762970.432975:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000003ce7ffbd 17ffffc0001001 3:0 ffff9360110dc2a0 2805 lru 00000080:00200000:2.0:1789762970.432991:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008565f94f 1781: 1< 0 - - > 2< 7294976 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.432996:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000048cd8bb3 00000080:00200000:2.0:1789762970.433002:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 246. 00000080:00200000:2.0:1789762970.433003:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2806 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433023:0:16278:0:(rw26.c:871:ll_write_end()) pos 11493376, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433054:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cecb65b7[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2806] 00000080:00200000:2.0:1789762970.433061:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000cf0fc811 17ffffc0001021 2:0 ffff9360110dce00 2806 lru 00000080:00200000:2.0:1789762970.433077:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000008c800c5c 1782: 1< 0 - - > 2< 7299072 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433084:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cecb65b7 00000080:00200000:2.0:1789762970.433087:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 247. 00000080:00200000:2.0:1789762970.433092:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2807 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433108:0:16278:0:(rw26.c:871:ll_write_end()) pos 11497472, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433116:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000d06efbf4[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2807] 00000080:00200000:2.0:1789762970.433121:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000008d0a297e 17ffffc0001001 3:0 ffff9360110dc1c0 2807 lru 00000080:00200000:2.0:1789762970.433126:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000003563764e 1783: 1< 0 - - > 2< 7303168 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433128:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000d06efbf4 00000080:00200000:2.0:1789762970.433129:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 248. 00000080:00200000:2.0:1789762970.433131:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2808 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433145:0:16278:0:(rw26.c:871:ll_write_end()) pos 11501568, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433157:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000030edd5b3[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2808] 00000080:00200000:2.0:1789762970.433179:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000e0ced5d3 17ffffc0001001 3:0 ffff9360110dcc40 2808 lru 00000080:00200000:2.0:1789762970.433213:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000713d15dc 1784: 1< 0 - - > 2< 7307264 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433218:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000030edd5b3 00000080:00200000:2.0:1789762970.433221:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 249. 00000080:00200000:2.0:1789762970.433225:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2809 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433237:0:16278:0:(rw26.c:871:ll_write_end()) pos 11505664, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433241:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000068434226[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2809] 00000080:00200000:2.0:1789762970.433243:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000037fcb832 17ffffc0001001 3:0 ffff9360110dca80 2809 lru 00000080:00200000:2.0:1789762970.433248:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000009da1c170 1785: 1< 0 - - > 2< 7311360 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433250:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000068434226 00000080:00200000:2.0:1789762970.433251:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 250. 00000080:00200000:2.0:1789762970.433254:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2810 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433269:0:16278:0:(rw26.c:871:ll_write_end()) pos 11509760, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433277:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000172e2677[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2810] 00000080:00200000:2.0:1789762970.433290:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000fcbd62c0 17ffffc0001001 3:0 ffff9360110dc700 2810 lru 00000080:00200000:2.0:1789762970.433308:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@00000000dedc751d 1786: 1< 0 - - > 2< 7315456 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433313:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000172e2677 00000080:00200000:2.0:1789762970.433327:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 251. 00000080:00200000:2.0:1789762970.433329:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2811 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433342:0:16278:0:(rw26.c:871:ll_write_end()) pos 11513856, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433347:0:16278:0:(rw26.c:929:ll_write_end()) page@000000005be568bf[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2811] 00000080:00200000:2.0:1789762970.433349:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000ee571021 17ffffc0001001 3:0 ffff9360110dc8c0 2811 lru 00000080:00200000:2.0:1789762970.433353:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000000feded58 1787: 1< 0 - - > 2< 7319552 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 13631488 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433355:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000005be568bf 00000080:00200000:2.0:1789762970.433356:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 252. 00000080:00200000:2.0:1789762970.433358:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2812 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433366:0:16278:0:(rw26.c:871:ll_write_end()) pos 11517952, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433370:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000fd9ca781[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2812] 00000080:00200000:2.0:1789762970.433374:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000dfe5e0a0 17ffffc0001001 3:0 ffff9360110dc7e0 2812 lru 00000080:00200000:2.0:1789762970.433381:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000056369700 1788: 1< 0 - - > 2< 7323648 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 11927552 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433384:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000fd9ca781 00000080:00200000:2.0:1789762970.433386:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 253. 00000080:00200000:2.0:1789762970.433392:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2813 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433414:0:16278:0:(rw26.c:871:ll_write_end()) pos 11522048, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433423:0:16278:0:(rw26.c:929:ll_write_end()) page@0000000059dfb78d[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2813] 00000080:00200000:2.0:1789762970.433425:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @00000000544165fa 17ffffc0001001 3:0 ffff9360110dc9a0 2813 lru 00000080:00200000:2.0:1789762970.433430:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000005cfd3a62 1789: 1< 0 - - > 2< 7327744 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 11927552 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433434:0:16278:0:(rw26.c:929:ll_write_end()) end page@0000000059dfb78d 00000080:00200000:2.0:1789762970.433439:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 254. 00000080:00200000:2.0:1789762970.433443:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2814 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433458:0:16278:0:(rw26.c:871:ll_write_end()) pos 11526144, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433461:0:16278:0:(rw26.c:929:ll_write_end()) page@00000000cec1daaa[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2814] 00000080:00200000:2.0:1789762970.433464:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @0000000012defb8c 17ffffc0001001 3:0 ffff9360110dc540 2814 lru 00000080:00200000:2.0:1789762970.433468:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@000000009baa1fd0 1790: 1< 0 - - > 2< 7331840 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 11927552 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433471:0:16278:0:(rw26.c:929:ll_write_end()) end page@00000000cec1daaa 00000080:00200000:2.0:1789762970.433472:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 255. 00000080:00200000:2.0:1789762970.433479:0:16278:0:(rw26.c:676:ll_write_begin()) Writing 2815 of 0 to 4096 bytes 00000080:00200000:2.0:1789762970.433491:0:16278:0:(rw26.c:871:ll_write_end()) pos 11530240, len 4096, copied 4096 00000080:00200000:2.0:1789762970.433495:0:16278:0:(rw26.c:929:ll_write_end()) page@000000003ea8bf35[3 00000000e9cce0bc 2 1 00000000329647cc 0 0 2815] 00000080:00200000:2.0:1789762970.433497:0:16278:0:(rw26.c:929:ll_write_end()) vmpage @000000002a698290 17ffffc0001001 3:0 ffff9360110dcb60 2815 lru 00000080:00200000:2.0:1789762970.433501:0:16278:0:(rw26.c:929:ll_write_end()) osc-page@0000000003429040 1791: 1< 0 - - > 2< 7335936 0 0 0x0 0x0 | 0000000085c3e132 000000008e07ea11 > 3< 0 0 > 4< 0 4 8 11927552 - | - - - - > 5< - - - - | 0 - | 0 - -> 00000080:00200000:2.0:1789762970.433503:0:16278:0:(rw26.c:929:ll_write_end()) end page@000000003ea8bf35 00000080:00200000:2.0:1789762970.433504:0:16278:0:(rw26.c:929:ll_write_end()) queued cl_page: 256. 00000080:00200000:2.0:1789762970.433507:0:16278:0:(vvp_io.c:1175:vvp_io_write_commit()) commit async pages: 256, from 0, to 4096 prio 0 00000008:00010000:2.0:1789762970.433551:0:16278:0:(osc_cache.c:858:osc_extent_find()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 4/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1789762970.433638:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.433655:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.433754:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.433889:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.433917:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.433956:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434053:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434180:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434212:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434286:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434349:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434441:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434456:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434554:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434568:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434663:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434681:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 15, dirtied 15 00000080:00200000:2.0:1789762970.434694:0:16278:0:(vvp_io.c:1088:vvp_set_batch_dirty()) mapping ffff936018e15d38, count 1, dirtied 1 00000080:00200000:2.0:1789762970.434697:0:16278:0:(vvp_io.c:1198:vvp_io_write_commit()) Committed 256 pages 1048576 bytes, tot: 1048576 00000080:00200000:2.0:1789762970.434700:0:16278:0:(vvp_io.c:1358:vvp_io_write_start()) nrs_w_oleg337-client.virtnet: write bytes 0, result: 1048576 00000020:00010000:2.0:1789762970.434720:0:16278:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3ac68 (ffff936008bea368/0) at cl_lock_release():245 00000020:00010000:2.0:1789762970.434723:0:16278:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3ac68 (ffff936008bea368/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762970.434726:0:16278:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061428b0 (ffff936008bea758/1) at cl_lock_cancel():142 00010000:00010000:2.0:1789762970.434739:0:16278:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 5/0,1 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x810020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762970.434750:0:16278:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1789762970.434761:0:16278:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3ac68 (ffff936008bea368/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762970.434765:0:16278:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061428b0 (ffff936008bea368/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762970.434771:0:16278:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:2.0:1789762970.434776:0:16278:0:(file.c:2205:ll_file_io_generic()) Range unlock [2560, 2815] 00000080:00200000:2.0:1789762970.434782:0:16278:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x21:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762970.434794:0:16278:0:(file.c:2238:ll_file_io_generic()) nrs_w_oleg337-client.virtnet: 2 io complete with rc: 0, result: 1048576, restart: 0 00000008:00010000:2.0:1789762970.434827:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1789762970.434841:0:255:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762970.435053:0:255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (6291456 6295551) ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1789762970.435454:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dd880 - object 0x2c0000401:3 offset 6291456<>7340032 00000008:00010000:2.0:1789762970.435475:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### extent: 00000000cc1772e2 ns: lustre-OST0001-osc-ffff936020592000 lock: 00000000322c1e5b/0xb758ac0edef9cb6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [4194304->5242879] (req 4194304->5242879) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aa4 expref: -99 pid: 16276 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000008:00010000:2.0:1789762970.435492:0:255:0:(osc_cache.c:1174:osc_extent_make_ready()) ### extent: 00000000cc1772e2 ns: lustre-OST0001-osc-ffff936020592000 lock: 00000000322c1e5b/0xb758ac0edef9cb6c lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [4194304->5242879] (req 4194304->5242879) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aa4 expref: -99 pid: 16276 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762970.435917:0:255:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4194304 4198399) ns: lustre-OST0001-osc-ffff936020592000 lock: 00000000322c1e5b/0xb758ac0edef9cb6c lrc: 3/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [4194304->5242879] (req 4194304->5242879) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177aa4 expref: -99 pid: 16276 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00100000:2.0:1789762970.436301:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dc380 - object 0x2c0000401:3 offset 4194304<>5242880 00000100:00100000:2.0:1789762970.436344:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067564160:192.168.203.137@tcp:4:dd.0 00000100:00100000:2.0:1789762970.436400:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762970.436406:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067564544:192.168.203.137@tcp:4:dd.0 00000008:00010000:2.0:1789762970.443608:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000008:00010000:2.0:1789762970.443780:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000e3ef0bf4 ns: lustre-OST0001-osc-ffff936020592000 lock: 0000000095d0cac7/0xb758ac0edef9cb5e lrc: 2/0,0 mode: PW/PW res: [0x2c0000401:0x3:0x0].0x0 rrc: 8 type: EXT [6291456->7340031] (req 6291456->7340031) gid 0 flags: 0x800020000020000 nid: local remote: 0xdf5a7ac59f177a96 expref: -99 pid: 16278 timeout: 0 lvb_type: 1 lru_score: 2 lru_type: 0 00000100:00100000:2.0:1789762970.444189:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067564160:192.168.203.137@tcp:4:dd.0 00000008:00010000:2.0:1789762970.546660:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000f9441bb8 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000788f1098/0xb758ac0edef9cb26 lrc: 5/0,0 mode: PW/PW res: [0x280000401:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a3b expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000008:00010000:2.0:1789762970.546994:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000f9441bb8 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000788f1098/0xb758ac0edef9cb26 lrc: 5/0,0 mode: PW/PW res: [0x280000401:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a3b expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000100:00100000:2.0:1789762970.547026:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067559552:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762970.665893:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:2.0:1789762970.946460:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000fc9eba75 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000788f1098/0xb758ac0edef9cb26 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a3b expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000008:00010000:2.0:1789762970.946699:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000fc9eba75 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000788f1098/0xb758ac0edef9cb26 lrc: 3/0,0 mode: PW/PW res: [0x280000401:0x9:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a3b expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000100:00100000:2.0:1789762970.946724:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067559808:192.168.203.137@tcp:4:dd.500 00000008:00010000:2.0:1789762971.347294:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000bf67cbc8 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000008a09ab9b/0xb758ac0edef9cb2d lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a49 expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000008:00010000:2.0:1789762971.347657:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000bf67cbc8 ns: lustre-OST0001-osc-ffff936020592000 lock: 000000008a09ab9b/0xb758ac0edef9cb2d lrc: 5/0,0 mode: PW/PW res: [0x2c0000401:0x2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a49 expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000100:00100000:2.0:1789762971.347698:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067560064:192.168.203.137@tcp:4:dd.500 00000008:00010000:2.0:1789762971.546908:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### extent: 00000000fe71ea6f ns: lustre-OST0001-osc-ffff936020592000 lock: 000000008a09ab9b/0xb758ac0edef9cb2d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a49 expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000008:00010000:2.0:1789762971.547303:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### extent: 00000000fe71ea6f ns: lustre-OST0001-osc-ffff936020592000 lock: 000000008a09ab9b/0xb758ac0edef9cb2d lrc: 4/0,0 mode: PW/PW res: [0x2c0000401:0x2:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177a49 expref: -99 pid: 16257 timeout: 0 lvb_type: 1 lru_score: 16 lru_type: 1 00000100:00100000:2.0:1789762971.547347:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067560192:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762971.956478:0:16286:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-OST0000-osc-ffff936008b5a800 00010000:00010000:2.0:1789762971.956530:0:16286:0:(ldlm_resource.c:195:lru_size_store()) dropping all unused locks from namespace lustre-OST0000-osc-ffff936020592000 00010000:00010000:2.0:1789762971.956553:0:16286:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1789762972.031670:0:16307:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908),name=/ 00000002:00010000:2.0:1789762972.031684:0:16307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789762972.031694:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1789762972.031704:0:16307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00000080:00200000:2.0:1789762972.031710:0:16307:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789762972.031716:0:16307:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) 00000080:00010000:2.0:1789762972.031720:0:16307:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) for lock 0xb758ac0edef9cb03 00000080:00010000:2.0:1789762972.031724:0:16307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb03 from it ffffb03e0499bbf8 00010000:00010000:2.0:1789762972.031729:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 14 lru_type: 1 00010000:00010000:2.0:1789762972.031736:0:16307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1789762972.031750:0:16307:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908), inode mode 41ed mask 1 00000080:00200000:2.0:1789762972.031756:0:16307:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d77k.sanityn, flags=25 00000080:00200000:2.0:1789762972.031761:0:16307:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x1e:0x0](ffff936018d9aa08), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.031765:0:16307:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=nrs_r_oleg337-client.virtnet, flags=265 00000080:00200000:2.0:1789762972.031770:0:16307:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x20:0x0](ffff936018d9e3c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1789762972.031775:0:16307:0:(file.c:1090:ll_file_open()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000002:00010000:2.0:1789762972.031789:0:16307:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000402:0x20:0x0]) in obj [0x200000402:0x1e:0x0], intent: open flags 0240100001 00010000:00010000:2.0:1789762972.032026:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: --/CR res: [0x200000402:0x1e:0x0].0x0 bits 0x0/0x0 rrc: 4 type: IBT flags: 0x10000000000000 pid: 16307 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.032033:0:16307:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: --/CR res: [0x200000402:0x1e:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 16307 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1789762972.032040:0:16307:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff936020592000: modify RPC slot 2 is allocated opc 101, max 7 00000100:00100000:2.0:1789762972.032046:0:16307:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9360061dd180 x1876702067565824/t0(0) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/131856 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1789762972.032055:0:16307:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: --/CR res: [0x200000402:0x1e:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT flags: 0x0 pid: 16307 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789762972.032067:0:16307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067565824:192.168.203.137@tcp:101:dd.0 00000100:00100000:2.0:1789762972.032197:0:16307:0:(client.c:2715:ptlrpc_set_wait()) set ffff936006e70f00 going to sleep for 11 seconds 00000100:00100000:2.0:1789762972.033752:0:16307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067565824:192.168.203.137@tcp:101:dd.0 00010000:00010000:2.0:1789762972.033767:0:16307:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: --/CR res: [0x200000402:0x1e:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033784:0:16307:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: --/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033846:0:16307:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: --/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033876:0:16307:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033886:0:16307:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033910:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 5/2,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033939:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 5/2,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.033949:0:16307:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 4/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789762972.033973:0:16307:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x1 disposition=0x2b, status=0 req@ffff9360061dd180 x1876702067565824/t4294967459(4294967459) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/688 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1789762972.033989:0:16307:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9360061dd180 x1876702067565824/t4294967459(4294967459) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/688 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00010000:2.0:1789762972.034000:0:16307:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open, lvb_len: 80 ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1789762972.034031:0:16307:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034059:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cba00/0xb758ac0edef9cb1f lrc: 2/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xdf5a7ac59f177a2d expref: -99 pid: 16257 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762972.034071:0:16307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cba00/0xb758ac0edef9cb1f lrc: 3/2,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xdf5a7ac59f177a2d expref: -99 pid: 16257 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762972.034086:0:16307:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 6 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034096:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034116:0:16307:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 5 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034138:0:16307:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000080:00200000:2.0:1789762972.034145:0:16307:0:(namei.c:952:ll_intent_lock()) intent lock 1 on i1 [0x200000402:0x1e:0x0] suppgids 0 -1: rc 0 00010000:00010000:2.0:1789762972.034179:0:6357:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00010000:2.0:1789762972.034187:0:6357:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034196:0:6357:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9360421cac00 already unused, calling callback (ffffffffc1a9a460) 00010000:00010000:2.0:1789762972.034204:0:6357:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034216:0:6357:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034236:0:6357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 3/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 5 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034293:0:6357:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cba00/0xb758ac0edef9cb1f lrc: 4/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 4 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xdf5a7ac59f177a2d expref: -99 pid: 16257 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762972.034364:0:6357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 2/0,0 mode: --/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034371:0:6357:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1789762972.034400:0:6357:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034407:0:6357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cb81 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x1b rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177ac7 expref: -99 pid: 16307 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.034424:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762972.034428:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762972.034431:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762972.034440:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789762972.034455:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067565952:192.168.203.137@tcp:103:ldlm_bl.0 00000080:00200000:2.0:1789762972.034525:0:16307:0:(file.c:6843:ll_layout_conf()) [0x200000402:0x20:0x0]: layout version change: 0 -> 0 00000080:00010000:2.0:1789762972.034530:0:16307:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000402:0x20:0x0](ffff936018d9e3c8) for lock 0xb758ac0edef9cb1f 00000080:00010000:2.0:1789762972.034535:0:16307:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb1f from it ffffb03e0499bbf0 00010000:00010000:2.0:1789762972.034538:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cba00/0xb758ac0edef9cb1f lrc: 3/1,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xdf5a7ac59f177a2d expref: -99 pid: 16257 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762972.034547:0:16307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff9360421cba00/0xb758ac0edef9cb1f lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x20:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xdf5a7ac59f177a2d expref: -99 pid: 16257 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000002:00100000:2.0:1789762972.034569:0:16307:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9360061dd180 x1876702067565824/t4294967459(4294967459) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 600/688 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000002:00100000:2.0:1789762972.034596:0:16307:0:(mdc_request.c:964:mdc_close()) @@@ matched open req@ffff9360061dd180 x1876702067565824/t4294967459(4294967459) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 600/688 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000020:00100000:2.0:1789762972.034606:0:16307:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0000-mdc-ffff936020592000: modify RPC slot 1 is allocated opc 35, max 7 00000100:00100000:2.0:1789762972.034617:0:16307:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9360061ded80 x1876702067566336/t0(0) o35->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:23/10 lens 392/624 e 0 to 0 dl 0 ref 1 fl New:QU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00000100:00100000:2.0:1789762972.034635:0:16307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067566336:192.168.203.137@tcp:35:dd.0 00000100:00100000:2.0:1789762972.034681:0:16307:0:(client.c:2715:ptlrpc_set_wait()) set ffff936006e70f00 going to sleep for 11 seconds 00000100:00100000:2.0:1789762972.035397:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067565952:192.168.203.137@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1789762972.035458:0:16307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067566336:192.168.203.137@tcp:35:dd.0 00000002:00100000:2.0:1789762972.035475:0:16307:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff9360061dd180 x1876702067565824/t4294967459(4294967459) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 600/688 e 0 to 0 dl 1789762988 ref 2 fl Complete:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000080:00200008:2.0:1789762972.035658:0:16307:0:(file.c:5214:ll_file_seek()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1789762972.035665:0:16307:0:(file.c:5245:ll_file_seek()) COMPLETED file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1789762972.035720:0:16307:0:(file.c:2483:do_file_read_iter()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1789762972.035726:0:16307:0:(rw.c:1925:ll_readpage()) fast read pgno: 0 00000080:00200000:2.0:1789762972.035732:0:16307:0:(file.c:2090:ll_file_io_generic()) nrs_r_oleg337-client.virtnet: read ppos: 0, bytes: 1048576 00000080:00200000:2.0:1789762972.035735:0:16307:0:(file.c:1848:ll_io_set_mirror()) nrs_r_oleg337-client.virtnet: desiginated mirror: 0 00000020:00200000:2.0:1789762972.035740:0:16307:0:(cl_io.c:214:cl_io_rw_init()) header@0000000038a8cbf0[0x110, 1, [0x200000402:0x20:0x0] hash] 00000020:00200000:2.0:1789762972.035743:0:16307:0:(cl_io.c:214:cl_io_rw_init()) io range: 1 [0, 1048576) 0 0 00000080:00200000:2.0:1789762972.035746:0:16307:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x20:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00020000:00200000:2.0:1789762972.035756:0:16307:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1789762972.035761:0:16307:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.035771:0:16307:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 0 [0, 255] 00000008:00010000:2.0:1789762972.035781:0:16307:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1789762972.035784:0:16307:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870730, osc lock ffff9360183de5f0, flags 0x0 00010000:00010000:2.0:1789762972.035794:0:16307:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff936042041c00 type 11 mode 6 res 10737419265/9 (0 1048575) 00010000:00010000:2.0:1789762972.037143:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 3/1,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->0] (req 0->0) gid 0 flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.037150:0:16307:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x0 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 3/1,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.037158:0:16307:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 3/1,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->1048575] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789762972.037169:0:16307:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067566592:192.168.203.137@tcp:101:dd.0 00000100:00100000:2.0:1789762972.037210:0:16307:0:(client.c:2715:ptlrpc_set_wait()) set ffff936006e70f00 going to sleep for 11 seconds 00000100:00100000:2.0:1789762972.037991:0:16307:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16307:1876702067566592:192.168.203.137@tcp:101:dd.0 00010000:00010000:2.0:1789762972.038009:0:16307:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/1,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.038018:0:16307:0:(ldlm_resource.c:1750:ldlm_resource_add_lock()) ### About to add this lock ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.038026:0:16307:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.038034:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000008:00010000:2.0:1789762972.038048:0:16307:0:(osc_lock.c:187:osc_lock_lvb_update()) ### acquired size 8388608, setting rss=8388608; kms=8388608, end=18446744073709551615 ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x10020000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.038057:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 5/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.038063:0:16307:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000080:00200000:2.0:1789762972.038075:0:16307:0:(vvp_io.c:835:vvp_io_read_start()) nrs_r_oleg337-client.virtnet: read [0, 1048576) 00000080:00200000:2.0:1789762972.038083:0:16307:0:(vvp_io.c:194:vvp_prep_size()) [0x200000402:0x20:0x0] updating i_size 12582912 00000080:00200000:2.0:1789762972.038088:0:16307:0:(vvp_io.c:893:vvp_io_read_start()) count:1048576 iocb pos:0 00000080:00200000:2.0:1789762972.041209:0:16310:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908),name=/ 00000002:00010000:2.0:1789762972.041226:0:16310:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789762972.041237:0:16310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1789762972.041248:0:16310:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1789762972.041261:0:16310:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789762972.041267:0:16310:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) 00000080:00010000:2.0:1789762972.041273:0:16310:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) for lock 0xb758ac0edef9cb03 00000080:00010000:2.0:1789762972.041288:0:16310:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb03 from it ffffb03e049b3bf8 00010000:00010000:2.0:1789762972.041293:0:16310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 4/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00010000:00010000:2.0:1789762972.041333:0:16310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 15 lru_type: 1 00000080:00200000:2.0:1789762972.041341:0:16310:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908), inode mode 41ed mask 1 00000080:00200000:2.0:1789762972.041349:0:16310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d77k.sanityn, flags=25 00000080:00200000:2.0:1789762972.041358:0:16310:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x1e:0x0](ffff936018d9aa08), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.041365:0:16310:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=nrs_r_oleg337-client.virtnet, flags=265 00000080:00200000:2.0:1789762972.041373:0:16310:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x20:0x0](ffff936018d9e3c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1789762972.041381:0:16310:0:(file.c:1090:ll_file_open()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200008:2.0:1789762972.041415:0:16310:0:(file.c:5214:ll_file_seek()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1789762972.041421:0:16310:0:(file.c:5245:ll_file_seek()) COMPLETED file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1789762972.041478:0:16310:0:(file.c:2483:do_file_read_iter()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1789762972.041946:0:16308:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908),name=/ 00000002:00010000:2.0:1789762972.041956:0:16308:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789762972.041963:0:16308:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1789762972.041971:0:16308:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00000080:00200000:2.0:1789762972.041976:0:16308:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789762972.041981:0:16308:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) 00000080:00010000:2.0:1789762972.041984:0:16308:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) for lock 0xb758ac0edef9cb03 00000080:00010000:2.0:1789762972.041987:0:16308:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb03 from it ffffb03e049a3bf8 00010000:00010000:2.0:1789762972.041991:0:16308:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 16 lru_type: 1 00010000:00010000:2.0:1789762972.042002:0:16308:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 17 lru_type: 1 00000080:00200000:2.0:1789762972.042009:0:16308:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908), inode mode 41ed mask 1 00000080:00200000:2.0:1789762972.042012:0:16308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d77k.sanityn, flags=25 00000080:00200000:2.0:1789762972.042015:0:16308:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x1e:0x0](ffff936018d9aa08), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.042018:0:16308:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=nrs_r_oleg337-client.virtnet, flags=265 00000080:00200000:2.0:1789762972.042023:0:16308:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x20:0x0](ffff936018d9e3c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1789762972.042027:0:16308:0:(file.c:1090:ll_file_open()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200008:2.0:1789762972.042051:0:16308:0:(file.c:5214:ll_file_seek()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1789762972.042055:0:16308:0:(file.c:5245:ll_file_seek()) COMPLETED file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1789762972.042106:0:16308:0:(file.c:2483:do_file_read_iter()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], ppos: 0, count: 1048576 00010000:00010000:2.0:1789762972.044744:0:16307:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 000000005109984a/0xb758ac0edef9cb88 lrc: 4/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177adc expref: -99 pid: 16306 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.044757:0:16307:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (4096 8191) ns: lustre-OST0000-osc-ffff936020592000 lock: 000000005109984a/0xb758ac0edef9cb88 lrc: 4/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177adc expref: -99 pid: 16306 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.044911:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 000000005109984a/0xb758ac0edef9cb88 lrc: 6/2,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0xdf5a7ac59f177adc expref: -99 pid: 16306 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.044919:0:16307:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff936020592000 lock: 000000005109984a/0xb758ac0edef9cb88 lrc: 5/1,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0xdf5a7ac59f177adc expref: -99 pid: 16306 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00000020:00010000:2.0:1789762972.050503:0:16307:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.050508:0:16307:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.050511:0:16307:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870730 (ffff936008beacf8/1) at cl_lock_cancel():142 00010000:00010000:2.0:1789762972.050516:0:16307:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 4/1,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x810020000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.050525:0:16307:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x800020000000000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00000020:00010000:2.0:1789762972.050537:0:16307:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.050539:0:16307:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870730 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.050544:0:16307:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000080:00200000:2.0:1789762972.050553:0:16307:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x20:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762972.050556:0:16307:0:(file.c:2238:ll_file_io_generic()) nrs_r_oleg337-client.virtnet: 1 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762972.050559:0:16307:0:(file.c:2324:ll_file_io_generic()) iot: 1, result: 1048576 00000080:00200008:2.0:1789762972.050574:0:16307:0:(file.c:479:ll_file_release()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200008:2.0:1789762972.061588:0:16310:0:(file.c:479:ll_file_release()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200000:2.0:1789762972.064074:0:16313:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908),name=/ 00000002:00010000:2.0:1789762972.064084:0:16313:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789762972.064094:0:16313:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1789762972.064104:0:16313:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00000080:00200000:2.0:1789762972.064111:0:16313:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789762972.064116:0:16313:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) 00000080:00010000:2.0:1789762972.064119:0:16313:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) for lock 0xb758ac0edef9cb03 00000080:00010000:2.0:1789762972.064123:0:16313:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb03 from it ffffb03e049cbbf8 00010000:00010000:2.0:1789762972.064127:0:16313:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 24 lru_type: 1 00010000:00010000:2.0:1789762972.064135:0:16313:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 25 lru_type: 1 00000080:00200000:2.0:1789762972.064146:0:16313:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908), inode mode 41ed mask 1 00000080:00200000:2.0:1789762972.064150:0:16313:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d77k.sanityn, flags=25 00000080:00200000:2.0:1789762972.064155:0:16313:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x1e:0x0](ffff936018d9aa08), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.064159:0:16313:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=nrs_r_oleg337-client.virtnet, flags=265 00000080:00200000:2.0:1789762972.064161:0:16313:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x20:0x0](ffff936018d9e3c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1789762972.064166:0:16313:0:(file.c:1090:ll_file_open()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200008:2.0:1789762972.064191:0:16313:0:(file.c:5214:ll_file_seek()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1789762972.064196:0:16313:0:(file.c:5245:ll_file_seek()) COMPLETED file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1789762972.064254:0:16313:0:(file.c:2483:do_file_read_iter()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], ppos: 0, count: 1048576 00000080:00200008:2.0:1789762972.067451:0:16308:0:(file.c:479:ll_file_release()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200000:2.0:1789762972.068584:0:16315:0:(file.c:6191:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908),name=/ 00000002:00010000:2.0:1789762972.068596:0:16315:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1789762972.068604:0:16315:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1789762972.068616:0:16315:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00000080:00200000:2.0:1789762972.068622:0:16315:0:(namei.c:952:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1789762972.068627:0:16315:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) 00000080:00010000:2.0:1789762972.068630:0:16315:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff936018d30908) for lock 0xb758ac0edef9cb03 00000080:00010000:2.0:1789762972.068635:0:16315:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cb03 from it ffffb03e049d3bf8 00010000:00010000:2.0:1789762972.068639:0:16315:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 26 lru_type: 1 00010000:00010000:2.0:1789762972.068644:0:16315:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936010e8d800/0xb758ac0edef9cb03 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f1779af expref: -99 pid: 16252 timeout: 0 lvb_type: 0 lru_score: 27 lru_type: 1 00000080:00200000:2.0:1789762972.068654:0:16315:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff936018d30908), inode mode 41ed mask 1 00000080:00200000:2.0:1789762972.068657:0:16315:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=d77k.sanityn, flags=25 00000080:00200000:2.0:1789762972.068661:0:16315:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x1e:0x0](ffff936018d9aa08), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.068665:0:16315:0:(dcache.c:276:ll_revalidate_dentry()) VFS Op:name=nrs_r_oleg337-client.virtnet, flags=265 00000080:00200000:2.0:1789762972.068670:0:16315:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x200000402:0x20:0x0](ffff936018d9e3c8), inode mode 81a4 mask 44 00000080:00200008:2.0:1789762972.068675:0:16315:0:(file.c:1090:ll_file_open()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000080:00200008:2.0:1789762972.068699:0:16315:0:(file.c:5214:ll_file_seek()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR 00000080:00200008:2.0:1789762972.068703:0:16315:0:(file.c:5245:ll_file_seek()) COMPLETED file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], offset: 0, type: SEEK_CUR, rc = 0 00000080:00200008:2.0:1789762972.068764:0:16315:0:(file.c:2483:do_file_read_iter()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0], ppos: 0, count: 1048576 00000080:00200008:2.0:1789762972.076297:0:16313:0:(file.c:479:ll_file_release()) START file nrs_r_oleg337-client.virtnet:[0x200000402:0x20:0x0](ffff936018d9e3c8), flags 100000 00000100:00100000:2.0:1789762972.079384:0:16320:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16320:1876702067566976:192.168.203.137@tcp:35:dd.0 00000002:00100000:2.0:1789762972.079408:0:16320:0:(mdc_request.c:832:mdc_free_open()) @@@ free open request, rq_replay=0 req@ffff936020dac700 x1876702067565568/t4294967460(4294967460) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 600/688 e 0 to 0 dl 1789762988 ref 1 fl Complete:RQU/604/ffffffff rc 0/-1 job:'dd.0' uid:0 gid:0 projid:0 00000100:00100000:2.0:1789762972.394240:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067569664:192.168.203.137@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1789762972.395552:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067569664:192.168.203.137@tcp:103:ldlm_bl.0 00010000:00010000:2.0:1789762972.438972:0:6357:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789762972.438985:0:6357:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439010:0:6357:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff936001482e00 already unused, calling callback (ffffffffc1a9a460) 00010000:00010000:2.0:1789762972.439014:0:6357:0:(ldlm_request.c:1732:ldlm_cli_convert()) ### client lock convert START ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439020:0:6357:0:(ldlm_request.c:1742:ldlm_cli_convert()) ### client lock convert END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x400000000 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439027:0:6357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 3/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439086:0:6357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 2/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439090:0:6357:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1789762972.439124:0:6357:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 1/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439129:0:6357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482e00/0xb758ac0edef9cbce lrc: 0/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x1 rrc: 3 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bfb expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.439159:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762972.439163:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762972.439166:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762972.473451:0:16340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16340:1876702067572096:192.168.203.137@tcp:101:dd.500 00010000:00010000:2.0:1789762972.473473:0:16340:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473490:0:16340:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473496:0:16340:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473506:0:16340:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473514:0:16340:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473523:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 5/2,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473528:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 5/2,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473534:0:16340:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789762972.473542:0:16340:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff936010ef6a00 x1876702067572096/t0(0) o101->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 584/584 e 0 to 0 dl 1789762988 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1789762972.473552:0:16340:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473560:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 2/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 4 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1789762972.473569:0:16340:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 2/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1789762972.473575:0:16340:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473580:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473584:0:16340:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936021291600/0xb758ac0edef9cbe3 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 3 type: IBT gid 0 flags: 0x10000400000000 nid: local remote: 0xdf5a7ac59f177c41 expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473604:0:16340:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00000002:00010000:2.0:1789762972.473617:0:16340:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: ,[0x0:0x0:0x0]) in obj [0x240000402:0x19:0x0], intent: lookup flags 0 00010000:00010000:2.0:1789762972.473649:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 3/1,0 mode: --/CR res: [0x240000402:0x19:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473657:0:16340:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 3/1,0 mode: --/CR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.473685:0:16340:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 3/1,0 mode: --/CR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT flags: 0x0 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789762972.473712:0:16340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16340:1876702067572224:192.168.203.137@tcp:101:dd.500 00000100:00100000:2.0:1789762972.473789:0:16340:0:(client.c:2715:ptlrpc_set_wait()) set ffff936006e706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789762972.474874:0:16340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16340:1876702067572224:192.168.203.137@tcp:101:dd.500 00010000:00010000:2.0:1789762972.474887:0:16340:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 4/1,0 mode: --/CR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474895:0:16340:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 4/1,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474904:0:16340:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474915:0:16340:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474925:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 5/2,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474935:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 5/2,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474944:0:16340:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789762972.474956:0:16340:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x10 disposition=0xb, status=0 req@ffff9360061ded80 x1876702067572224/t0(0) o101->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 576/584 e 0 to 0 dl 1789762988 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1789762972.474968:0:16340:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.474976:0:16340:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff936007c82400 type 13 mode 4 res 9663677442/25 (0 0) 00000080:00200000:2.0:1789762972.474988:0:16340:0:(namei.c:952:ll_intent_lock()) intent lock 16 on i1 [0x200000007:0x1:0x0] suppgids -1 -1: rc 0 00000080:00200000:2.0:1789762972.475006:0:16340:0:(llite_lib.c:3119:ll_update_inode()) inode=[0x240000402:0x19:0x0], updating i_size 16384 00000080:00200000:2.0:1789762972.475009:0:16340:0:(namei.c:147:ll_iget()) got inode: [0x240000402:0x19:0x0](ffff936018e17448): rc = 0 00000080:00010000:2.0:1789762972.475013:0:16340:0:(llite_internal.h:1981:ll_set_lock_data()) setting l_data to inode [0x240000402:0x19:0x0](ffff936018e17448) for remote lock 0xb758ac0edef9cbea 00000080:00010000:2.0:1789762972.475017:0:16340:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x240000402:0x19:0x0](ffff936018e17448) for lock 0xb758ac0edef9cbb9 00000080:00010000:2.0:1789762972.475026:0:16340:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000402:0x19:0x0](ffff936018e17448) 00000080:00010000:2.0:1789762972.475028:0:16340:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cbb9 from it ffffb03e0496bac8 00010000:00010000:2.0:1789762972.475031:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1789762972.475038:0:16340:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000080:00010000:2.0:1789762972.475047:0:16340:0:(dcache.c:164:ll_intent_drop_lock()) releasing remote lock with cookie 0xb758ac0edef9cbea from it ffffb03e0496bac8 00010000:00010000:2.0:1789762972.475049:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.475054:0:16340:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1789762972.475068:0:16340:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x240000402:0x19:0x0](ffff936018e17448), inode mode 41ff mask 1 00000080:00200000:2.0:1789762972.475083:0:16340:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x240000402:0x19:0x0](ffff936018e17448), inode mode 41ff mask 3 00000080:00200000:2.0:1789762972.475088:0:16340:0:(namei.c:1412:ll_atomic_open()) VFS Op:name=tbf, dir=[0x240000402:0x19:0x0](ffff936018e17448), file ffff936007d4dc80, open_flags c241, mode 81b6 opened 0 00000080:00200000:2.0:1789762972.475093:0:16340:0:(namei.c:1072:ll_lookup_it()) VFS Op:name=tbf, dir=[0x240000402:0x19:0x0](ffff936018e17448), intent=open|creat 00000002:00010000:2.0:1789762972.475111:0:16340:0:(mdc_locks.c:1508:mdc_intent_lock()) (name: tbf,[0x240000402:0x1a:0x0]) in obj [0x240000402:0x19:0x0], intent: open|creat flags 0141102 00010000:00010000:2.0:1789762972.475136:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CW) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/0,1 mode: --/CW res: [0x240000402:0x19:0x0].0x0 bits 0x0/0x0 rrc: 3 type: IBT flags: 0x10000000000000 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.475141:0:16340:0:(ldlm_request.c:1107:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/0,1 mode: --/CW res: [0x240000402:0x19:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00100000:2.0:1789762972.475152:0:16340:0:(genops.c:2428:obd_get_mod_rpc_slot()) lustre-MDT0001-mdc-ffff936020592000: modify RPC slot 1 is allocated opc 101, max 7 00000100:00100000:2.0:1789762972.475158:0:16340:0:(client.c:820:ptlrpc_reassign_next_xid()) @@@ reassign xid req@ffff9360061ded80 x1876702067572608/t0(0) o101->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/66320 e 0 to 0 dl 0 ref 1 fl New:PQU/400/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:0 00010000:00010000:2.0:1789762972.475165:0:16340:0:(ldlm_request.c:1194:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/0,1 mode: --/CW res: [0x240000402:0x19:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT flags: 0x0 pid: 16340 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1789762972.475175:0:16340:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16340:1876702067572608:192.168.203.137@tcp:101:dd.500 00000100:00100000:2.0:1789762972.475272:0:16340:0:(client.c:2715:ptlrpc_set_wait()) set ffff936006e706c0 going to sleep for 11 seconds 00000100:00100000:2.0:1789762972.476990:0:16340:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ded80 pname:cluuid:pid:xid:nid:opc:job dd:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:16340:1876702067572608:192.168.203.137@tcp:101:dd.500 00010000:00010000:2.0:1789762972.477001:0:16340:0:(ldlm_request.c:759:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/0,1 mode: --/CW res: [0x240000402:0x19:0x0].0x0 bits 0x2/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477013:0:16340:0:(ldlm_request.c:774:ldlm_cli_enqueue_fini()) ### client-side enqueue, new resource ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/0,1 mode: --/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477019:0:16340:0:(ldlm_request.c:787:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/0,1 mode: --/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477024:0:16340:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/0,1 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477030:0:16340:0:(ldlm_request.c:841:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/0,1 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477036:0:16340:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 5/1,1 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477045:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CW) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 5/1,1 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477050:0:16340:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 4/1,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:2.0:1789762972.477056:0:16340:0:(mdc_locks.c:835:mdc_finish_enqueue()) @@@ op=0x3 disposition=0x37, status=0 req@ffff9360061ded80 x1876702067572608/t4294967471(4294967471) o101->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/664 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000002:00100000:2.0:1789762972.477066:0:16340:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9360061ded80 x1876702067572608/t4294967471(4294967471) o101->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 648/664 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000002:00010000:2.0:1789762972.477075:0:16340:0:(mdc_locks.c:939:mdc_finish_enqueue()) ### layout lock returned by: open|creat, lvb_len: 56 ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00010000:2.0:1789762972.477083:0:16340:0:(mdc_locks.c:1359:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477089:0:16340:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff936007c82400 type 13 mode 4 res 9663677442/26 (0 0) 00000080:00200000:2.0:1789762972.477092:0:16340:0:(namei.c:952:ll_intent_lock()) intent lock 3 on i1 [0x240000402:0x19:0x0] suppgids -1 -1: rc 0 00000080:00200000:2.0:1789762972.477097:0:16340:0:(namei.c:1628:ll_create_it()) VFS Op:name=tbf, dir=[0x240000402:0x19:0x0](ffff936018e17448), intent=open|creat 00000080:00200000:2.0:1789762972.477115:0:16340:0:(llite_lib.c:3356:ll_read_inode2()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8) 00000080:00200000:2.0:1789762972.477181:0:16340:0:(file.c:6843:ll_layout_conf()) [0x240000402:0x1a:0x0]: layout version change: 4294967294 -> 0 00000080:00010000:2.0:1789762972.477185:0:16340:0:(namei.c:1594:ll_create_node()) setting l_ast_data to inode [0x240000402:0x1a:0x0](ffff936018d980c8) 00000080:00010000:2.0:1789762972.477187:0:16340:0:(llite_internal.h:1989:ll_set_lock_data()) setting l_data to inode [0x240000402:0x1a:0x0](ffff936018d980c8) for lock 0xb758ac0edef9cbf1 00000080:00010000:2.0:1789762972.477192:0:16340:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x240000402:0x1a:0x0](ffff936018d980c8) 00000080:00200008:2.0:1789762972.477202:0:16340:0:(file.c:1090:ll_file_open()) START file tbf:[0x240000402:0x1a:0x0](ffff936018d980c8), flags 141101 00000002:00100000:2.0:1789762972.477216:0:16340:0:(mdc_request.c:811:mdc_set_open_replay_data()) @@@ Set up open replay data req@ffff9360061ded80 x1876702067572608/t4294967471(4294967471) o101->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 576/664 e 0 to 0 dl 1789762988 ref 2 fl Complete:RPQU/604/0 rc 0/0 job:'dd.500' uid:500 gid:500 projid:0 00000080:00010000:2.0:1789762972.477252:0:16340:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0xb758ac0edef9cbf1 from it ffff936007037cc0 00010000:00010000:2.0:1789762972.477256:0:16340:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 3/1,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x10020000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1789762972.477262:0:16340:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x20000000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1789762972.477274:0:16340:0:(file.c:6672:ll_inode_permission()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), inode mode 81a4 mask 40 00000080:00200008:2.0:1789762972.478666:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 0, count: 1048576 00000080:00200000:2.0:1789762972.478675:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 0, bytes: 1048576 00000080:00200000:2.0:1789762972.478682:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762972.478689:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762972.478691:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [0, 1048576) 0 0 00000080:00200000:2.0:1789762972.478696:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762972.478707:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [0, 255] 00020000:00200000:2.0:1789762972.478714:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 0 chunk: [0, 1048576) 1048576, 1048576 00020000:00200000:2.0:1789762972.478717:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.478732:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [0, 255] 00000008:00010000:2.0:1789762972.478744:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762972.478749:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762972.478757:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [0, 1048576) append 0 append lockpos 0 00000080:00200000:2.0:1789762972.478767:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=0=0x0, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762972.479073:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762972.479076:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 1048576 00000020:00010000:2.0:1789762972.479083:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.479086:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.479088:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.479091:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.479093:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.479099:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762972.479105:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762972.479120:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625600 00000008:00100000:2.0:1789762972.479332:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dca80 - object 0x280000400:15 offset 0<>1048576 00000080:00200000:2.0:1789762972.520468:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [0, 255] 00000080:00200000:2.0:1789762972.520488:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762972.520494:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762972.520510:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 1048576 i_blocks 1 00000020:00200000:2.0:1789762972.520513:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762972.520526:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762972.520775:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 1048576, count: 1048576 00000080:00200000:2.0:1789762972.520782:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 1048576, bytes: 1048576 00000080:00200000:2.0:1789762972.520792:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762972.520850:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762972.520855:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [1048576, 2097152) 0 0 00000080:00200000:2.0:1789762972.520861:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762972.520875:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [256, 511] 00020000:00200000:2.0:1789762972.520883:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 1048576 chunk: [1048576, 2097152) 2097152, 1048576 00020000:00200000:2.0:1789762972.520887:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.520907:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [256, 511] 00000008:00010000:2.0:1789762972.520922:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762972.520928:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762972.520938:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [1048576, 2097152) append 0 append lockpos 0 00000080:00200000:2.0:1789762972.520949:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=1048576=0x100000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762972.521232:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762972.521235:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 2097152 00000020:00010000:2.0:1789762972.521242:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.521246:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.521248:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.521250:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.521253:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.521259:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762972.521264:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762972.521281:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625d80 00000008:00100000:2.0:1789762972.521560:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dfb80 - object 0x280000400:15 offset 1048576<>2097152 00000080:00200000:2.0:1789762972.536439:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [256, 511] 00000080:00200000:2.0:1789762972.536456:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762972.536463:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762972.536475:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 2097152 i_blocks 2051 00000020:00200000:2.0:1789762972.536480:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762972.536489:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762972.536666:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 2097152, count: 1048576 00000080:00200000:2.0:1789762972.536685:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 2097152, bytes: 1048576 00000080:00200000:2.0:1789762972.536693:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762972.536698:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762972.536700:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [2097152, 3145728) 0 0 00000080:00200000:2.0:1789762972.536703:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762972.536713:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [512, 767] 00020000:00200000:2.0:1789762972.536719:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 2097152 chunk: [2097152, 3145728) 3145728, 1048576 00020000:00200000:2.0:1789762972.536722:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.536745:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [512, 767] 00000008:00010000:2.0:1789762972.536753:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762972.536756:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762972.536762:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [2097152, 3145728) append 0 append lockpos 0 00000080:00200000:2.0:1789762972.536770:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=2097152=0x200000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762972.537121:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762972.537123:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 3145728 00000020:00010000:2.0:1789762972.537129:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.537131:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.537133:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.537135:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.537138:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.537142:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762972.537146:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762972.537159:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a6256c0 00000008:00100000:2.0:1789762972.537388:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 2097152<>3145728 00000080:00200000:2.0:1789762972.737582:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [512, 767] 00000080:00200000:2.0:1789762972.737597:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762972.737604:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762972.737620:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 3145728 i_blocks 4103 00000020:00200000:2.0:1789762972.737625:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762972.737637:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762972.737913:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 3145728, count: 1048576 00000080:00200000:2.0:1789762972.737932:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 3145728, bytes: 1048576 00000080:00200000:2.0:1789762972.737940:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762972.737947:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762972.737949:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [3145728, 4194304) 0 0 00000080:00200000:2.0:1789762972.737953:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762972.737966:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [768, 1023] 00020000:00200000:2.0:1789762972.737974:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 3145728 chunk: [3145728, 4194304) 4194304, 1048576 00020000:00200000:2.0:1789762972.737977:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.737993:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [768, 1023] 00000008:00010000:2.0:1789762972.738006:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762972.738010:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762972.738020:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [3145728, 4194304) append 0 append lockpos 0 00000080:00200000:2.0:1789762972.738031:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=3145728=0x300000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762972.738277:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762972.738280:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 4194304 00000020:00010000:2.0:1789762972.738287:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.738290:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.738293:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.738297:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.738310:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.738319:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762972.738326:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762972.738352:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625540 00000008:00100000:2.0:1789762972.738590:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 3145728<>4194304 00000080:00200000:2.0:1789762972.936104:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [768, 1023] 00000080:00200000:2.0:1789762972.936135:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762972.936140:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762972.936154:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 4194304 i_blocks 6151 00000020:00200000:2.0:1789762972.936158:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762972.936171:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762972.936436:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 4194304, count: 1048576 00000080:00200000:2.0:1789762972.936444:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 4194304, bytes: 1048576 00000080:00200000:2.0:1789762972.936451:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762972.936459:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762972.936461:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [4194304, 5242880) 0 0 00000080:00200000:2.0:1789762972.936464:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762972.936477:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [1024, 1279] 00020000:00200000:2.0:1789762972.936484:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 4194304 chunk: [4194304, 5242880) 5242880, 1048576 00020000:00200000:2.0:1789762972.936487:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762972.936503:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [1024, 1279] 00000008:00010000:2.0:1789762972.936517:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762972.936521:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762972.936532:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [4194304, 5242880) append 0 append lockpos 0 00000080:00200000:2.0:1789762972.936542:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=4194304=0x400000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762972.936893:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762972.936895:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 5242880 00000020:00010000:2.0:1789762972.936902:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762972.936905:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.936907:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762972.936910:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762972.936912:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762972.936918:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762972.936924:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762972.936934:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625300 00000008:00100000:2.0:1789762972.937214:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061ddf80 - object 0x280000400:15 offset 4194304<>5242880 00000080:00200000:2.0:1789762973.138032:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [1024, 1279] 00000080:00200000:2.0:1789762973.138049:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762973.138055:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762973.138069:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 5242880 i_blocks 8199 00000020:00200000:2.0:1789762973.138072:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762973.138083:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762973.138325:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 5242880, count: 1048576 00000080:00200000:2.0:1789762973.138332:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 5242880, bytes: 1048576 00000080:00200000:2.0:1789762973.138340:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762973.138348:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762973.138350:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [5242880, 6291456) 0 0 00000080:00200000:2.0:1789762973.138354:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762973.138369:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [1280, 1535] 00020000:00200000:2.0:1789762973.138377:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 5242880 chunk: [5242880, 6291456) 6291456, 1048576 00020000:00200000:2.0:1789762973.138381:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762973.138398:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [1280, 1535] 00000008:00010000:2.0:1789762973.138413:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762973.138418:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762973.138430:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [5242880, 6291456) append 0 append lockpos 0 00000080:00200000:2.0:1789762973.138445:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=5242880=0x500000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762973.138761:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762973.138764:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 6291456 00000020:00010000:2.0:1789762973.138772:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762973.138775:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.138777:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.138780:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762973.138783:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762973.138789:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762973.138794:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762973.138845:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762973.139094:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061ddf80 - object 0x280000400:15 offset 5242880<>6291456 00000100:00100000:2.0:1789762973.139129:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789762973.139142:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573376:192.168.203.137@tcp:4:dd.500 00000008:00010000:2.0:1789762973.335257:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762973.335271:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762973.335275:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762973.335276:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762973.335278:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762973.335279:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762973.335280:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762973.335282:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762973.335283:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762973.335284:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762973.335285:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762973.335286:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762973.335287:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762973.335288:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762973.335289:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762973.335290:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762973.335291:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762973.335292:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762973.335293:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762973.335294:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762973.335306:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762973.335308:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762973.335309:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762973.335310:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762973.335313:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762973.335314:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762973.335316:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762973.335317:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762973.335318:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762973.335319:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762973.335320:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762973.335321:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762973.335322:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762973.335323:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762973.335324:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762973.335325:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762973.335326:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762973.335328:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762973.335329:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762973.335330:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762973.335331:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762973.335332:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762973.335334:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762973.335335:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762973.335336:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762973.335337:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762973.335338:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762973.335339:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762973.335340:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762973.335341:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762973.335342:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762973.335343:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762973.335344:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762973.335345:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762973.335346:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762973.335347:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762973.335349:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762973.335350:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762973.335352:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762973.335353:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762973.335354:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762973.335355:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762973.335356:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762973.335357:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762973.335358:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762973.335359:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762973.335360:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762973.335361:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762973.335362:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762973.335363:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762973.335364:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762973.335365:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762973.335367:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762973.335368:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762973.335369:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762973.335370:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762973.335371:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762973.335372:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762973.335373:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762973.335374:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762973.335375:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762973.335376:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762973.335377:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762973.335378:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762973.335379:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762973.335380:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762973.335382:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762973.335383:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762973.335384:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762973.335385:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762973.335387:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762973.335388:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762973.335389:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762973.335391:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762973.335392:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762973.335393:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762973.335394:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762973.335395:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762973.335396:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762973.335397:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762973.335398:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762973.335399:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762973.335400:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762973.335401:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762973.335402:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762973.335404:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762973.335405:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762973.335405:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762973.335406:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762973.335408:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762973.335409:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762973.335410:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762973.335411:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762973.335412:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762973.335413:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762973.335414:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762973.335415:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762973.335416:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762973.335417:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762973.335418:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762973.335419:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762973.335421:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762973.335422:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762973.335423:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762973.335424:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762973.335425:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762973.335426:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762973.335427:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762973.335428:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762973.335429:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762973.335431:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762973.335432:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762973.335433:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762973.335434:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762973.335435:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762973.335436:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762973.335437:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762973.335438:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762973.335439:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762973.335440:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762973.335442:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762973.335442:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762973.335443:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762973.335444:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762973.335446:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762973.335447:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762973.335448:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762973.335449:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762973.335450:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762973.335451:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762973.335452:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762973.335453:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762973.335455:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762973.335456:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762973.335457:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762973.335459:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762973.335460:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762973.335461:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762973.335462:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762973.335463:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762973.335464:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762973.335465:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762973.335466:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762973.335467:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762973.335468:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762973.335470:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762973.335471:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762973.335472:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762973.335473:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762973.335474:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762973.335475:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762973.335476:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762973.335477:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762973.335478:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762973.335480:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762973.335481:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762973.335482:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762973.335483:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762973.335484:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762973.335485:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762973.335486:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762973.335487:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762973.335488:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762973.335489:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762973.335491:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762973.335492:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762973.335493:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762973.335494:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762973.335495:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762973.335496:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762973.335497:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762973.335498:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762973.335499:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762973.335500:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762973.335501:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762973.335502:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762973.335503:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762973.335504:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762973.335505:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762973.335506:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762973.335508:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762973.335509:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762973.335510:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762973.335511:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762973.335512:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762973.335513:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762973.335514:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762973.335515:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762973.335517:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762973.335518:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762973.335519:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762973.335520:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762973.335522:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762973.335522:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762973.335523:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762973.335524:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762973.335526:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762973.335527:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762973.335528:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762973.335529:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762973.335530:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762973.335532:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762973.335533:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762973.335534:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762973.335535:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762973.335536:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762973.335537:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762973.335538:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762973.335539:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762973.335540:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762973.335541:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762973.335542:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762973.335543:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762973.335544:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762973.335545:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762973.335546:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762973.335548:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762973.335549:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762973.335550:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762973.335551:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762973.335552:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762973.335553:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762973.335554:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762973.335555:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762973.335556:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762973.335557:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762973.335558:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762973.335559:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762973.335560:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762973.335561:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762973.335563:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762973.335564:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762973.335565:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762973.335566:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762973.335568:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762973.335569:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762973.335571:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762973.335659:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762973.335680:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762973.335705:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573376:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762973.537183:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [1536, 1791] 00000080:00200000:2.0:1789762973.537200:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762973.537205:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762973.537217:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 7340032 i_blocks 12295 00000020:00200000:2.0:1789762973.537220:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762973.537231:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762973.537441:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 7340032, count: 1048576 00000080:00200000:2.0:1789762973.537448:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 7340032, bytes: 1048576 00000080:00200000:2.0:1789762973.537456:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762973.537463:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762973.537465:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [7340032, 8388608) 0 0 00000080:00200000:2.0:1789762973.537469:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762973.537482:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [1792, 2047] 00020000:00200000:2.0:1789762973.537489:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 7340032 chunk: [7340032, 8388608) 8388608, 1048576 00020000:00200000:2.0:1789762973.537492:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762973.537508:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [1792, 2047] 00000008:00010000:2.0:1789762973.537521:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762973.537526:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762973.537535:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [7340032, 8388608) append 0 append lockpos 0 00000080:00200000:2.0:1789762973.537546:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=7340032=0x700000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762973.537731:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762973.537734:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 8388608 00000020:00010000:2.0:1789762973.537740:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762973.537742:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.537745:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.537747:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762973.537749:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762973.537763:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762973.537772:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762973.537784:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762973.538048:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 7340032<>8388608 00000100:00100000:2.0:1789762973.538076:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789762973.538088:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573632:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762973.737457:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968157 gen 4 00000008:00010000:2.0:1789762973.737516:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762973.737523:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762973.737526:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762973.737527:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762973.737528:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762973.737530:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762973.737531:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762973.737532:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762973.737533:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762973.737534:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762973.737536:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762973.737537:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762973.737538:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762973.737540:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762973.737541:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762973.737543:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762973.737544:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762973.737545:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762973.737547:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762973.737548:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762973.737549:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762973.737550:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762973.737551:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762973.737552:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762973.737553:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762973.737554:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762973.737555:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762973.737556:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762973.737557:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762973.737558:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762973.737559:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762973.737560:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762973.737561:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762973.737562:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762973.737563:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762973.737564:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762973.737565:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762973.737566:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762973.737567:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762973.737569:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762973.737570:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762973.737571:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762973.737572:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762973.737573:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762973.737574:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762973.737575:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762973.737576:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762973.737578:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762973.737579:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762973.737580:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762973.737582:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762973.737583:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762973.737584:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762973.737585:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762973.737587:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762973.737589:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762973.737590:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762973.737592:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762973.737594:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762973.737596:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762973.737598:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762973.737600:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762973.737602:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762973.737604:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762973.737605:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762973.737607:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762973.737609:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762973.737611:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762973.737613:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762973.737615:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762973.737618:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762973.737620:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762973.737622:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762973.737624:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762973.737626:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762973.737627:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762973.737629:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762973.737631:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762973.737633:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762973.737636:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762973.737638:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762973.737640:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762973.737642:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762973.737654:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762973.737656:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762973.737657:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762973.737658:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762973.737659:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762973.737660:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762973.737661:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762973.737662:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762973.737664:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762973.737665:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762973.737666:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762973.737667:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762973.737668:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762973.737669:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762973.737670:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762973.737671:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762973.737672:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762973.737673:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762973.737674:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762973.737675:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762973.737676:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762973.737677:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762973.737678:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762973.737679:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762973.737680:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762973.737681:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762973.737682:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762973.737683:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762973.737685:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762973.737686:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762973.737687:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762973.737689:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762973.737690:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762973.737691:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762973.737692:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762973.737693:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762973.737694:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762973.737695:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762973.737696:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762973.737697:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762973.737698:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762973.737699:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762973.737700:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762973.737701:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762973.737702:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762973.737703:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762973.737704:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762973.737705:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762973.737706:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762973.737707:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762973.737709:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762973.737710:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762973.737711:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762973.737712:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762973.737713:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762973.737714:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762973.737715:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762973.737727:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762973.737728:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762973.737729:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762973.737731:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762973.737732:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762973.737734:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762973.737735:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762973.737736:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762973.737737:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762973.737738:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762973.737739:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762973.737740:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762973.737741:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762973.737742:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762973.737743:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762973.737745:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762973.737746:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762973.737747:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762973.737748:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762973.737749:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762973.737750:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762973.737751:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762973.737752:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762973.737753:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762973.737754:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762973.737755:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762973.737757:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762973.737758:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762973.737759:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762973.737760:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762973.737761:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762973.737762:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762973.737763:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762973.737764:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762973.737765:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762973.737766:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762973.737768:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762973.737769:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762973.737770:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762973.737771:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762973.737773:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762973.737774:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762973.737775:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762973.737776:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762973.737777:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762973.737778:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762973.737779:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762973.737780:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762973.737781:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762973.737782:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762973.737783:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762973.737784:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762973.737785:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762973.737786:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762973.737787:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762973.737788:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762973.737789:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762973.737790:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762973.737791:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762973.737792:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762973.737793:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762973.737795:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762973.737796:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762973.737829:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762973.737831:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762973.737833:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762973.737834:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762973.737836:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762973.737837:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762973.737839:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762973.737840:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762973.737842:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762973.737843:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762973.737844:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762973.737845:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762973.737846:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762973.737847:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762973.737849:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762973.737850:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762973.737851:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762973.737852:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762973.737854:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762973.737855:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762973.737856:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762973.737857:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762973.737859:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762973.737860:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762973.737862:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762973.737863:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762973.737865:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762973.737866:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762973.737868:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762973.737869:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762973.737870:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762973.737871:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762973.737872:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762973.737874:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762973.737875:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762973.737876:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762973.737879:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762973.737880:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762973.737881:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762973.737883:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762973.737883:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762973.737885:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762973.737886:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762973.737887:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762973.737888:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762973.737889:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762973.737890:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762973.737891:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762973.737892:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762973.737893:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762973.737894:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762973.737895:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762973.737897:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762973.737898:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762973.737996:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762973.738014:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762973.738040:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573632:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762973.738125:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1789762973.936772:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [2048, 2303] 00000080:00200000:2.0:1789762973.936789:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762973.936794:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762973.936827:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 9437184 i_blocks 16391 00000020:00200000:2.0:1789762973.936830:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762973.936842:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762973.937058:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 9437184, count: 1048576 00000080:00200000:2.0:1789762973.937065:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 9437184, bytes: 1048576 00000080:00200000:2.0:1789762973.937073:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762973.937081:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762973.937083:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [9437184, 10485760) 0 0 00000080:00200000:2.0:1789762973.937087:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762973.937101:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [2304, 2559] 00020000:00200000:2.0:1789762973.937108:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 9437184 chunk: [9437184, 10485760) 10485760, 1048576 00020000:00200000:2.0:1789762973.937112:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762973.937128:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [2304, 2559] 00000008:00010000:2.0:1789762973.937142:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762973.937147:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762973.937158:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [9437184, 10485760) append 0 append lockpos 0 00000080:00200000:2.0:1789762973.937169:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=9437184=0x900000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762973.937422:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762973.937425:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 10485760 00000020:00010000:2.0:1789762973.937432:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762973.937435:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.937437:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762973.937440:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762973.937442:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762973.937448:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762973.937453:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762973.937467:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762973.937714:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 9437184<>10485760 00000100:00100000:2.0:1789762973.937746:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789762973.937758:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573888:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762974.137010:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968159 gen 4 00000008:00010000:2.0:1789762974.137093:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762974.137115:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762974.137119:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762974.137121:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762974.137123:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762974.137137:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762974.137143:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762974.137145:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762974.137147:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762974.137149:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762974.137151:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762974.137153:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762974.137155:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762974.137157:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762974.137159:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762974.137160:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762974.137163:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762974.137165:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762974.137167:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762974.137169:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762974.137171:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762974.137173:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762974.137175:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762974.137177:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762974.137179:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762974.137180:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762974.137182:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762974.137184:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762974.137186:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762974.137188:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762974.137190:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762974.137192:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762974.137194:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762974.137196:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762974.137198:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762974.137200:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762974.137202:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762974.137205:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762974.137207:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762974.137210:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762974.137212:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762974.137214:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762974.137216:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762974.137218:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762974.137220:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762974.137221:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762974.137223:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762974.137225:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762974.137227:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762974.137229:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762974.137230:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762974.137233:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762974.137235:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762974.137236:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762974.137238:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762974.137241:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762974.137243:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762974.137245:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762974.137247:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762974.137248:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762974.137251:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762974.137252:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762974.137254:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762974.137256:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762974.137258:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762974.137260:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762974.137261:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762974.137263:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762974.137265:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762974.137267:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762974.137269:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762974.137271:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762974.137273:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762974.137275:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762974.137277:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762974.137279:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762974.137281:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762974.137283:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762974.137284:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762974.137286:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762974.137288:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762974.137303:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762974.137306:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762974.137308:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762974.137310:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762974.137313:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762974.137315:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762974.137316:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762974.137319:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762974.137321:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762974.137323:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762974.137325:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762974.137327:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762974.137329:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762974.137331:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762974.137333:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762974.137335:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762974.137336:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762974.137338:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762974.137339:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762974.137341:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762974.137354:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762974.137355:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762974.137357:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762974.137359:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762974.137361:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762974.137363:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762974.137364:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762974.137366:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762974.137368:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762974.137370:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762974.137371:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762974.137374:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762974.137376:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762974.137377:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762974.137379:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762974.137381:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762974.137383:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762974.137385:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762974.137386:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762974.137388:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762974.137390:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762974.137392:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762974.137394:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762974.137397:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762974.137399:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762974.137401:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762974.137403:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762974.137404:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762974.137406:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762974.137408:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762974.137410:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762974.137412:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762974.137414:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762974.137416:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762974.137418:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762974.137420:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762974.137433:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762974.137435:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762974.137437:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762974.137439:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762974.137441:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762974.137443:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762974.137444:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762974.137446:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762974.137448:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762974.137450:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762974.137451:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762974.137453:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762974.137455:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762974.137456:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762974.137458:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762974.137460:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762974.137462:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762974.137464:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762974.137466:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762974.137468:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762974.137469:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762974.137471:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762974.137473:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762974.137474:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762974.137476:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762974.137478:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762974.137480:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762974.137482:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762974.137484:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762974.137486:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762974.137487:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762974.137489:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762974.137491:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762974.137492:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762974.137494:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762974.137496:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762974.137497:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762974.137499:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762974.137502:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762974.137503:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762974.137505:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762974.137507:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762974.137509:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762974.137510:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762974.137512:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762974.137514:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762974.137516:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762974.137518:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762974.137520:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762974.137522:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762974.137524:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762974.137525:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762974.137527:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762974.137529:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762974.137531:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762974.137533:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762974.137535:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762974.137536:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762974.137538:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762974.137540:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762974.137542:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762974.137544:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762974.137546:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762974.137548:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762974.137550:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762974.137552:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762974.137554:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762974.137555:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762974.137557:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762974.137559:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762974.137561:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762974.137562:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762974.137564:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762974.137566:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762974.137568:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762974.137570:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762974.137571:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762974.137573:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762974.137575:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762974.137577:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762974.137579:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762974.137581:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762974.137582:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762974.137584:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762974.137586:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762974.137588:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762974.137590:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762974.137593:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762974.137595:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762974.137597:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762974.137599:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762974.137601:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762974.137603:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762974.137605:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762974.137607:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762974.137608:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762974.137610:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762974.137612:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762974.137614:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762974.137616:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762974.137617:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762974.137619:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762974.137621:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762974.137622:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762974.137624:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762974.137626:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762974.137627:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762974.137629:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762974.137631:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762974.137632:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762974.137634:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762974.137635:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762974.137637:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762974.137638:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762974.137640:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762974.137642:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762974.137644:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762974.137647:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762974.137649:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762974.137651:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762974.137778:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762974.137823:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762974.137867:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067573888:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762974.337449:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [2560, 2815] 00000080:00200000:2.0:1789762974.337467:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762974.337474:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762974.337488:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 11534336 i_blocks 20487 00000020:00200000:2.0:1789762974.337493:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762974.337506:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762974.337730:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 11534336, count: 1048576 00000080:00200000:2.0:1789762974.337738:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 11534336, bytes: 1048576 00000080:00200000:2.0:1789762974.337748:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762974.337756:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762974.337759:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [11534336, 12582912) 0 0 00000080:00200000:2.0:1789762974.337763:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762974.337779:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [2816, 3071] 00020000:00200000:2.0:1789762974.337787:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 11534336 chunk: [11534336, 12582912) 12582912, 1048576 00020000:00200000:2.0:1789762974.337792:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762974.337832:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [2816, 3071] 00000008:00010000:2.0:1789762974.337847:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762974.337853:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762974.337865:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [11534336, 12582912) append 0 append lockpos 0 00000080:00200000:2.0:1789762974.337879:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=11534336=0xb00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762974.338185:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762974.338189:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 12582912 00000020:00010000:2.0:1789762974.338198:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762974.338201:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762974.338204:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762974.338208:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762974.338211:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762974.338219:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762974.338226:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762974.338242:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762974.338566:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 11534336<>12582912 00000100:00100000:2.0:1789762974.338605:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789762974.338621:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067574144:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762974.539960:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968161 gen 4 00000008:00010000:2.0:1789762974.540021:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762974.540029:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762974.540032:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762974.540033:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762974.540034:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762974.540035:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762974.540036:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762974.540038:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762974.540039:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762974.540040:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762974.540041:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762974.540042:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762974.540043:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762974.540044:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762974.540045:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762974.540046:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762974.540047:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762974.540048:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762974.540049:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762974.540050:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762974.540051:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762974.540052:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762974.540053:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762974.540054:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762974.540055:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762974.540056:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762974.540057:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762974.540058:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762974.540059:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762974.540061:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762974.540063:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762974.540064:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762974.540065:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762974.540066:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762974.540067:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762974.540068:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762974.540069:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762974.540070:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762974.540071:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762974.540072:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762974.540073:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762974.540074:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762974.540076:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762974.540077:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762974.540077:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762974.540079:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762974.540080:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762974.540081:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762974.540082:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762974.540083:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762974.540084:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762974.540085:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762974.540086:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762974.540087:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762974.540088:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762974.540089:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762974.540090:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762974.540091:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762974.540092:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762974.540093:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762974.540094:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762974.540096:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762974.540098:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762974.540099:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762974.540101:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762974.540102:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762974.540103:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762974.540104:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762974.540105:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762974.540106:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762974.540107:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762974.540108:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762974.540109:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762974.540110:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762974.540111:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762974.540112:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762974.540113:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762974.540114:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762974.540115:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762974.540117:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762974.540118:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762974.540119:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762974.540120:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762974.540130:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762974.540131:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762974.540132:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762974.540133:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762974.540134:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762974.540135:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762974.540136:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762974.540137:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762974.540138:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762974.540139:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762974.540141:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762974.540143:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762974.540144:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762974.540146:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762974.540147:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762974.540148:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762974.540149:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762974.540151:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762974.540152:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762974.540153:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762974.540154:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762974.540155:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762974.540156:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762974.540157:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762974.540158:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762974.540159:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762974.540160:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762974.540161:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762974.540162:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762974.540163:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762974.540164:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762974.540165:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762974.540166:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762974.540167:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762974.540168:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762974.540169:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762974.540170:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762974.540171:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762974.540172:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762974.540173:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762974.540174:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762974.540175:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762974.540177:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762974.540178:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762974.540180:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762974.540182:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762974.540183:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762974.540184:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762974.540185:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762974.540186:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762974.540187:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762974.540188:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762974.540189:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762974.540190:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762974.540191:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762974.540192:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762974.540193:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762974.540194:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762974.540195:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762974.540196:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762974.540197:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762974.540198:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762974.540199:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762974.540200:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762974.540201:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762974.540203:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762974.540204:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762974.540205:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762974.540206:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762974.540207:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762974.540208:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762974.540209:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762974.540210:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762974.540211:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762974.540212:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762974.540214:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762974.540215:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762974.540217:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762974.540218:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762974.540219:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762974.540220:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762974.540221:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762974.540222:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762974.540224:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762974.540225:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762974.540226:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762974.540227:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762974.540228:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762974.540229:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762974.540230:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762974.540231:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762974.540232:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762974.540233:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762974.540234:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762974.540235:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762974.540236:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762974.540237:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762974.540238:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762974.540239:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762974.540241:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762974.540242:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762974.540243:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762974.540244:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762974.540245:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762974.540246:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762974.540247:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762974.540249:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762974.540250:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762974.540252:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762974.540253:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762974.540254:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762974.540255:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762974.540256:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762974.540257:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762974.540258:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762974.540259:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762974.540260:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762974.540261:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762974.540262:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762974.540263:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762974.540264:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762974.540265:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762974.540266:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762974.540267:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762974.540269:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762974.540270:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762974.540271:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762974.540272:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762974.540273:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762974.540274:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762974.540275:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762974.540276:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762974.540277:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762974.540278:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762974.540279:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762974.540281:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762974.540282:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762974.540283:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762974.540284:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762974.540286:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762974.540299:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762974.540300:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762974.540302:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762974.540303:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762974.540304:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762974.540305:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762974.540306:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762974.540307:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762974.540308:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762974.540309:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762974.540311:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762974.540312:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762974.540313:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762974.540315:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762974.540317:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762974.540319:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762974.540321:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762974.540322:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762974.540324:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762974.540325:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762974.540327:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762974.540328:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762974.540330:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762974.540339:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762974.540341:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762974.540343:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762974.540344:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762974.540345:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762974.540347:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762974.540348:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762974.540350:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762974.540352:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762974.540353:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762974.540354:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762974.540433:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762974.540449:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762974.540474:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067574144:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762974.739885:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [3072, 3327] 00000080:00200000:2.0:1789762974.739900:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762974.739906:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762974.739919:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 13631488 i_blocks 24583 00000020:00200000:2.0:1789762974.739922:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762974.739934:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762974.740160:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 13631488, count: 1048576 00000080:00200000:2.0:1789762974.740167:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 13631488, bytes: 1048576 00000080:00200000:2.0:1789762974.740175:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762974.740182:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762974.740185:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [13631488, 14680064) 0 0 00000080:00200000:2.0:1789762974.740188:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762974.740202:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [3328, 3583] 00020000:00200000:2.0:1789762974.740210:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 13631488 chunk: [13631488, 14680064) 14680064, 1048576 00020000:00200000:2.0:1789762974.740213:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762974.740231:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [3328, 3583] 00000008:00010000:2.0:1789762974.740255:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762974.740260:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762974.740271:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [13631488, 14680064) append 0 append lockpos 0 00000080:00200000:2.0:1789762974.740282:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=13631488=0xd00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762974.740549:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762974.740552:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 14680064 00000020:00010000:2.0:1789762974.740561:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762974.740564:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762974.740567:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762974.740570:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762974.740572:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762974.740578:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762974.740584:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762974.740597:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762974.740890:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 13631488<>14680064 00000100:00100000:2.0:1789762974.740925:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574400:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762974.937207:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968163 gen 4 00000008:00010000:2.0:1789762974.937346:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762974.937356:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762974.937359:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762974.937362:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762974.937364:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762974.937366:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762974.937368:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762974.937371:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762974.937373:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762974.937375:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762974.937377:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762974.937379:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762974.937381:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762974.937382:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762974.937384:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762974.937386:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762974.937388:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762974.937390:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762974.937391:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762974.937393:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762974.937395:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762974.937398:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762974.937400:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762974.937402:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762974.937404:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762974.937405:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762974.937407:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762974.937409:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762974.937415:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762974.937418:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762974.937420:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762974.937422:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762974.937424:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762974.937426:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762974.937429:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762974.937431:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762974.937433:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762974.937435:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762974.937437:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762974.937439:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762974.937441:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762974.937443:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762974.937445:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762974.937447:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762974.937449:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762974.937451:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762974.937453:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762974.937454:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762974.937456:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762974.937458:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762974.937459:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762974.937461:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762974.937463:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762974.937465:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762974.937467:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762974.937469:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762974.937472:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762974.937474:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762974.937476:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762974.937478:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762974.937480:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762974.937482:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762974.937484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762974.937486:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762974.937487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762974.937489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762974.937491:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762974.937493:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762974.937494:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762974.937496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762974.937498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762974.937499:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762974.937501:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762974.937503:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762974.937504:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762974.937506:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762974.937508:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762974.937510:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762974.937512:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762974.937513:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762974.937515:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762974.937517:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762974.937519:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762974.937520:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762974.937522:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762974.937524:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762974.937527:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762974.937529:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762974.937531:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762974.937533:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762974.937535:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762974.937537:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762974.937540:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762974.937542:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762974.937544:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762974.937545:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762974.937548:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762974.937549:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762974.937551:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762974.937552:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762974.937554:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762974.937556:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762974.937557:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762974.937559:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762974.937561:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762974.937563:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762974.937565:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762974.937566:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762974.937568:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762974.937570:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762974.937572:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762974.937574:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762974.937576:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762974.937578:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762974.937580:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762974.937582:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762974.937584:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762974.937586:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762974.937588:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762974.937591:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762974.937593:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762974.937595:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762974.937597:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762974.937598:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762974.937600:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762974.937602:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762974.937604:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762974.937606:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762974.937608:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762974.937610:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762974.937612:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762974.937614:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762974.937617:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762974.937619:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762974.937621:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762974.937623:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762974.937626:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762974.937628:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762974.937629:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762974.937631:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762974.937633:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762974.937635:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762974.937637:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762974.937638:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762974.937641:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762974.937643:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762974.937645:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762974.937647:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762974.937648:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762974.937650:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762974.937653:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762974.937655:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762974.937657:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762974.937659:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762974.937661:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762974.937662:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762974.937664:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762974.937666:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762974.937668:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762974.937670:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762974.937672:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762974.937674:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762974.937676:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762974.937679:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762974.937680:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762974.937682:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762974.937685:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762974.937687:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762974.937689:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762974.937691:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762974.937693:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762974.937695:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762974.937697:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762974.937699:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762974.937701:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762974.937703:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762974.937704:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762974.937706:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762974.937708:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762974.937710:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762974.937711:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762974.937714:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762974.937716:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762974.937718:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762974.937720:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762974.937722:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762974.937723:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762974.937725:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762974.937727:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762974.937729:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762974.937731:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762974.937734:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762974.937736:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762974.937738:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762974.937740:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762974.937742:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762974.937744:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762974.937746:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762974.937747:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762974.937749:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762974.937751:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762974.937753:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762974.937755:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762974.937757:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762974.937760:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762974.937762:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762974.937764:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762974.937766:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762974.937768:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762974.937771:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762974.937773:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762974.937775:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762974.937777:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762974.937779:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762974.937781:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762974.937783:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762974.937785:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762974.937787:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762974.937788:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762974.937790:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762974.937792:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762974.937793:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762974.937795:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762974.937826:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762974.937829:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762974.937831:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762974.937833:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762974.937834:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762974.937836:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762974.937837:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762974.937839:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762974.937841:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762974.937843:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762974.937845:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762974.937846:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762974.937848:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762974.937850:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762974.937852:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762974.937853:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762974.937855:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762974.937856:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762974.937858:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762974.937859:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762974.937861:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762974.937863:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762974.937865:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762974.937867:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762974.937869:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762974.937871:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762974.937872:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762974.937874:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762974.937876:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762974.937877:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762974.937879:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762974.937880:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762974.937882:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762974.937884:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762974.938000:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762974.938020:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762974.938051:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574400:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762974.939022:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762974.939038:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574528:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762975.135892:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968164 gen 4 00000008:00010000:2.0:1789762975.135953:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdc3c0 00000020:00200000:2.0:1789762975.135961:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762975.135964:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762975.135965:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762975.135967:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762975.135968:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762975.135969:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762975.135970:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762975.135971:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762975.135972:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762975.135973:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762975.135974:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762975.135976:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762975.135978:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762975.135979:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762975.135981:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762975.135983:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762975.135985:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762975.135986:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762975.135988:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762975.135990:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762975.135991:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762975.135993:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762975.135995:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762975.135998:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762975.136000:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762975.136002:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762975.136004:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762975.136006:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762975.136009:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762975.136011:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762975.136013:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762975.136015:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762975.136017:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762975.136020:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762975.136022:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762975.136024:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762975.136026:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762975.136028:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762975.136031:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762975.136033:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762975.136034:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762975.136036:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762975.136038:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762975.136040:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762975.136042:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762975.136043:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762975.136045:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762975.136046:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762975.136047:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762975.136048:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762975.136049:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762975.136050:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762975.136051:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762975.136052:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762975.136053:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762975.136054:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762975.136056:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762975.136057:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762975.136058:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762975.136059:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762975.136060:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762975.136061:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762975.136062:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762975.136063:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762975.136064:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762975.136065:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762975.136066:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762975.136067:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762975.136068:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762975.136069:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762975.136070:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762975.136071:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762975.136072:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762975.136073:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762975.136074:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762975.136076:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762975.136077:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762975.136079:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762975.136080:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762975.136082:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762975.136083:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762975.136084:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762975.136085:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762975.136086:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762975.136087:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762975.136088:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762975.136089:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762975.136090:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762975.136091:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762975.136092:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762975.136093:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762975.136094:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762975.136095:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762975.136096:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762975.136097:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762975.136098:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762975.136099:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762975.136100:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762975.136101:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762975.136102:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762975.136103:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762975.136104:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762975.136105:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762975.136106:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762975.136107:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762975.136108:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762975.136109:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762975.136111:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762975.136112:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762975.136113:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762975.136115:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762975.136116:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762975.136117:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762975.136118:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762975.136119:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762975.136120:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762975.136121:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762975.136122:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762975.136123:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762975.136124:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762975.136125:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762975.136126:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762975.136127:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762975.136129:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762975.136129:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762975.136130:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762975.136132:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762975.136133:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762975.136134:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762975.136135:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762975.136136:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762975.136137:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762975.136138:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762975.136139:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762975.136140:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762975.136141:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762975.136142:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762975.136143:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762975.136144:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762975.136146:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762975.136147:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762975.136148:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762975.136149:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762975.136150:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762975.136152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762975.136153:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762975.136154:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762975.136155:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762975.136156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762975.136157:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762975.136158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762975.136159:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762975.136160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762975.136161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762975.136162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762975.136163:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762975.136164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762975.136165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762975.136166:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762975.136167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762975.136168:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762975.136169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762975.136171:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762975.136172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762975.136173:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762975.136174:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762975.136175:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762975.136176:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762975.136177:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762975.136178:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762975.136179:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762975.136180:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762975.136181:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762975.136182:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762975.136184:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762975.136185:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762975.136186:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762975.136187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762975.136188:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762975.136189:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762975.136190:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762975.136191:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762975.136192:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762975.136193:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762975.136194:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762975.136195:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762975.136196:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762975.136197:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762975.136198:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762975.136199:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762975.136200:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762975.136201:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762975.136202:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762975.136203:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762975.136204:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762975.136205:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762975.136206:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762975.136207:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762975.136208:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762975.136209:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762975.136210:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762975.136211:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762975.136213:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762975.136214:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762975.136216:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762975.136217:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762975.136218:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762975.136219:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762975.136220:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762975.136222:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762975.136223:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762975.136224:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762975.136225:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762975.136226:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762975.136227:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762975.136228:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762975.136229:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762975.136230:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762975.136231:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762975.136232:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762975.136233:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762975.136234:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762975.136235:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762975.136236:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762975.136237:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762975.136238:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762975.136239:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762975.136240:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762975.136241:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762975.136242:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762975.136243:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762975.136244:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762975.136245:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762975.136246:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762975.136248:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762975.136249:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762975.136251:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762975.136252:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762975.136253:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762975.136255:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762975.136256:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762975.136257:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762975.136258:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762975.136259:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762975.136260:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762975.136261:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762975.136262:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762975.136263:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762975.136264:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762975.136264:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762975.136265:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762975.136267:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762975.136268:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762975.136269:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762975.136270:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762975.136271:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762975.136359:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762975.136377:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdc3c0 00000100:00100000:2.0:1789762975.136417:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574528:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762975.337015:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [3840, 4095] 00000080:00200000:2.0:1789762975.337034:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762975.337040:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762975.337054:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 16777216 i_blocks 30727 00000020:00200000:2.0:1789762975.337057:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762975.337076:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762975.337342:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 16777216, count: 1048576 00000080:00200000:2.0:1789762975.337351:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 16777216, bytes: 1048576 00000080:00200000:2.0:1789762975.337361:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762975.337371:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762975.337374:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [16777216, 17825792) 0 0 00000080:00200000:2.0:1789762975.337379:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762975.337398:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [4096, 4351] 00020000:00200000:2.0:1789762975.337407:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 16777216 chunk: [16777216, 17825792) 17825792, 1048576 00020000:00200000:2.0:1789762975.337413:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762975.337433:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4096, 4351] 00000008:00010000:2.0:1789762975.337452:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762975.337459:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762975.337472:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [16777216, 17825792) append 0 append lockpos 0 00000080:00200000:2.0:1789762975.337486:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=16777216=0x1000000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762975.337875:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762975.337878:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 17825792 00000020:00010000:2.0:1789762975.337887:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762975.337891:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762975.337895:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762975.337898:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762975.337902:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762975.337909:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762975.337916:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762975.337932:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762975.338249:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 16777216<>17825792 00000100:00100000:2.0:1789762975.338299:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762975.338315:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574784:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762975.537906:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968166 gen 4 00000008:00010000:2.0:1789762975.537960:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762975.537967:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762975.537970:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762975.537972:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762975.537973:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762975.537974:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762975.537975:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762975.537977:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762975.537978:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762975.537979:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762975.537980:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762975.537981:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762975.537982:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762975.537983:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762975.537984:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762975.537985:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762975.537986:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762975.537987:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762975.537988:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762975.537989:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762975.537990:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762975.537992:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762975.537993:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762975.537994:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762975.537995:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762975.537996:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762975.537997:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762975.537999:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762975.538000:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762975.538002:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762975.538003:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762975.538004:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762975.538006:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762975.538007:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762975.538008:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762975.538009:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762975.538010:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762975.538012:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762975.538013:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762975.538014:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762975.538015:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762975.538017:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762975.538018:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762975.538019:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762975.538020:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762975.538021:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762975.538022:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762975.538023:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762975.538024:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762975.538025:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762975.538026:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762975.538027:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762975.538028:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762975.538029:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762975.538030:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762975.538031:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762975.538032:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762975.538033:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762975.538034:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762975.538035:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762975.538036:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762975.538037:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762975.538038:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762975.538039:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762975.538040:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762975.538041:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762975.538042:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762975.538044:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762975.538045:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762975.538046:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762975.538048:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762975.538049:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762975.538050:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762975.538051:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762975.538052:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762975.538054:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762975.538055:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762975.538056:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762975.538057:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762975.538059:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762975.538060:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762975.538061:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762975.538062:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762975.538064:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762975.538065:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762975.538066:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762975.538067:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762975.538069:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762975.538070:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762975.538071:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762975.538072:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762975.538073:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762975.538075:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762975.538083:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762975.538085:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762975.538086:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762975.538087:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762975.538089:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762975.538090:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762975.538093:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762975.538095:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762975.538097:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762975.538099:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762975.538101:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762975.538103:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762975.538104:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762975.538105:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762975.538106:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762975.538107:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762975.538108:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762975.538109:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762975.538110:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762975.538111:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762975.538112:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762975.538114:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762975.538115:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762975.538116:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762975.538117:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762975.538118:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762975.538119:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762975.538120:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762975.538121:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762975.538122:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762975.538123:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762975.538124:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762975.538125:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762975.538126:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762975.538127:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762975.538128:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762975.538129:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762975.538130:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762975.538132:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762975.538133:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762975.538135:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762975.538136:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762975.538137:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762975.538138:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762975.538140:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762975.538141:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762975.538142:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762975.538143:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762975.538144:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762975.538145:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762975.538146:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762975.538147:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762975.538149:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762975.538150:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762975.538151:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762975.538152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762975.538153:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762975.538154:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762975.538155:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762975.538156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762975.538157:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762975.538158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762975.538159:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762975.538160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762975.538161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762975.538162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762975.538163:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762975.538164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762975.538165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762975.538166:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762975.538167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762975.538169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762975.538170:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762975.538171:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762975.538172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762975.538173:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762975.538174:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762975.538175:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762975.538176:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762975.538177:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762975.538179:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762975.538180:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762975.538181:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762975.538182:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762975.538182:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762975.538184:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762975.538185:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762975.538186:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762975.538187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762975.538188:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762975.538189:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762975.538190:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762975.538191:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762975.538192:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762975.538193:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762975.538195:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762975.538197:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762975.538199:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762975.538201:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762975.538202:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762975.538204:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762975.538206:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762975.538208:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762975.538210:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762975.538212:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762975.538214:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762975.538216:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762975.538217:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762975.538219:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762975.538221:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762975.538223:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762975.538224:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762975.538226:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762975.538228:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762975.538231:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762975.538232:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762975.538234:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762975.538236:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762975.538238:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762975.538240:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762975.538242:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762975.538244:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762975.538246:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762975.538248:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762975.538250:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762975.538252:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762975.538253:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762975.538255:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762975.538256:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762975.538257:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762975.538258:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762975.538259:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762975.538260:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762975.538261:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762975.538263:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762975.538264:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762975.538265:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762975.538266:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762975.538268:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762975.538269:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762975.538271:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762975.538273:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762975.538274:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762975.538276:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762975.538278:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762975.538295:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762975.538297:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762975.538299:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762975.538301:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762975.538303:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762975.538305:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762975.538307:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762975.538309:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762975.538311:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762975.538313:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762975.538315:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762975.538317:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762975.538320:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762975.538322:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762975.538323:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762975.538325:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762975.538326:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762975.538328:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762975.538329:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762975.538419:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762975.538438:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762975.538461:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067574784:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762975.739534:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [4352, 4607] 00000080:00200000:2.0:1789762975.739555:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762975.739562:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762975.739577:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 18874368 i_blocks 34823 00000020:00200000:2.0:1789762975.739581:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762975.739597:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762975.739896:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 18874368, count: 1048576 00000080:00200000:2.0:1789762975.739906:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18874368, bytes: 1048576 00000080:00200000:2.0:1789762975.739916:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762975.739924:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762975.739926:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18874368, 19922944) 0 0 00000080:00200000:2.0:1789762975.739930:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762975.739943:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [4608, 4863] 00020000:00200000:2.0:1789762975.739951:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18874368 chunk: [18874368, 19922944) 19922944, 1048576 00020000:00200000:2.0:1789762975.739954:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762975.739972:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4608, 4863] 00000008:00010000:2.0:1789762975.739986:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762975.739991:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762975.740002:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18874368, 19922944) append 0 append lockpos 0 00000080:00200000:2.0:1789762975.740013:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=18874368=0x1200000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762975.740220:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762975.740222:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 19922944 00000020:00010000:2.0:1789762975.740230:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762975.740233:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762975.740235:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762975.740238:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762975.740240:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762975.740246:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762975.740251:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762975.740267:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762975.740547:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 18874368<>19922944 00000100:00100000:2.0:1789762975.740585:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762975.740602:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067575040:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762975.938397:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968168 gen 4 00000008:00010000:2.0:1789762975.938470:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762975.938479:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762975.938484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762975.938487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762975.938489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762975.938492:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762975.938495:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762975.938496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762975.938498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762975.938500:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762975.938502:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762975.938504:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762975.938505:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762975.938507:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762975.938508:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762975.938510:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762975.938511:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762975.938513:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762975.938515:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762975.938517:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762975.938519:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762975.938520:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762975.938522:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762975.938524:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762975.938527:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762975.938528:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762975.938531:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762975.938533:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762975.938534:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762975.938536:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762975.938538:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762975.938540:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762975.938541:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762975.938543:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762975.938544:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762975.938546:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762975.938547:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762975.938549:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762975.938551:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762975.938553:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762975.938555:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762975.938557:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762975.938559:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762975.938561:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762975.938563:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762975.938565:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762975.938567:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762975.938568:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762975.938570:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762975.938572:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762975.938573:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762975.938575:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762975.938577:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762975.938579:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762975.938580:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762975.938582:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762975.938584:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762975.938586:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762975.938589:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762975.938591:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762975.938593:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762975.938595:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762975.938597:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762975.938599:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762975.938601:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762975.938603:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762975.938605:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762975.938606:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762975.938608:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762975.938610:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762975.938612:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762975.938614:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762975.938616:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762975.938618:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762975.938621:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762975.938623:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762975.938625:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762975.938627:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762975.938629:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762975.938631:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762975.938632:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762975.938634:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762975.938635:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762975.938637:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762975.938638:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762975.938639:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762975.938640:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762975.938641:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762975.938642:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762975.938643:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762975.938645:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762975.938646:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762975.938647:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762975.938649:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762975.938650:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762975.938651:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762975.938652:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762975.938653:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762975.938654:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762975.938663:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762975.938664:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762975.938665:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762975.938666:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762975.938667:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762975.938668:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762975.938669:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762975.938670:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762975.938671:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762975.938673:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762975.938674:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762975.938675:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762975.938677:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762975.938678:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762975.938679:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762975.938680:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762975.938681:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762975.938682:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762975.938683:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762975.938684:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762975.938685:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762975.938686:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762975.938687:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762975.938688:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762975.938690:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762975.938695:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762975.938697:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762975.938700:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762975.938702:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762975.938704:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762975.938706:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762975.938708:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762975.938710:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762975.938711:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762975.938713:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762975.938715:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762975.938717:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762975.938719:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762975.938722:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762975.938724:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762975.938726:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762975.938728:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762975.938730:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762975.938733:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762975.938735:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762975.938737:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762975.938739:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762975.938740:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762975.938742:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762975.938744:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762975.938746:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762975.938748:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762975.938749:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762975.938750:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762975.938752:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762975.938754:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762975.938757:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762975.938759:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762975.938761:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762975.938763:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762975.938765:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762975.938767:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762975.938769:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762975.938771:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762975.938773:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762975.938775:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762975.938777:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762975.938779:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762975.938781:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762975.938783:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762975.938784:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762975.938786:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762975.938788:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762975.938790:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762975.938792:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762975.938794:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762975.938823:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762975.938825:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762975.938826:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762975.938827:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762975.938829:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762975.938830:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762975.938831:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762975.938832:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762975.938834:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762975.938835:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762975.938837:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762975.938840:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762975.938842:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762975.938844:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762975.938845:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762975.938847:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762975.938849:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762975.938851:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762975.938853:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762975.938855:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762975.938857:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762975.938859:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762975.938861:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762975.938864:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762975.938866:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762975.938868:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762975.938871:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762975.938873:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762975.938876:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762975.938878:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762975.938880:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762975.938881:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762975.938883:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762975.938886:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762975.938887:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762975.938889:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762975.938890:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762975.938892:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762975.938893:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762975.938894:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762975.938895:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762975.938896:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762975.938897:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762975.938899:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762975.938900:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762975.938902:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762975.938903:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762975.938904:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762975.938905:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762975.938906:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762975.938907:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762975.938908:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762975.938909:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762975.938910:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762975.938911:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762975.938912:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762975.938913:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762975.938914:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762975.938915:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762975.938916:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762975.938918:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762975.938919:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762975.938920:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762975.938921:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762975.938922:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762975.938923:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762975.938924:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762975.938925:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762975.938926:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762975.938927:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762975.938928:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762975.938929:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762975.938930:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762975.938931:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762975.938932:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762975.938934:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762975.938935:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762975.938937:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762975.938938:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762975.938939:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762975.938940:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762975.938941:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762975.939020:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762975.939042:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762975.939070:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067575040:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762976.139250:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [4864, 5119] 00000080:00200000:2.0:1789762976.139271:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762976.139290:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762976.139309:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 20971520 i_blocks 38919 00000020:00200000:2.0:1789762976.139313:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762976.139339:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762976.139592:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 20971520, count: 1048576 00000080:00200000:2.0:1789762976.139599:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20971520, bytes: 1048576 00000080:00200000:2.0:1789762976.139607:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762976.139615:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762976.139617:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20971520, 22020096) 0 0 00000080:00200000:2.0:1789762976.139621:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762976.139634:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [5120, 5375] 00020000:00200000:2.0:1789762976.139641:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20971520 chunk: [20971520, 22020096) 22020096, 1048576 00020000:00200000:2.0:1789762976.139645:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762976.139662:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5120, 5375] 00000008:00010000:2.0:1789762976.139676:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762976.139693:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762976.139704:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20971520, 22020096) append 0 append lockpos 0 00000080:00200000:2.0:1789762976.139718:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=20971520=0x1400000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762976.139972:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762976.139975:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 22020096 00000020:00010000:2.0:1789762976.139982:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762976.139985:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762976.139987:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762976.139990:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762976.139992:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762976.139997:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762976.140003:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762976.140018:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762976.140335:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 20971520<>22020096 00000100:00100000:2.0:1789762976.140379:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762976.140395:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067576064:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762976.338791:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968170 gen 4 00000008:00010000:2.0:1789762976.338879:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762976.338886:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762976.338889:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762976.338891:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762976.338894:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762976.338896:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762976.338897:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762976.338899:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762976.338900:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762976.338901:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762976.338902:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762976.338903:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762976.338906:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762976.338907:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762976.338908:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762976.338909:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762976.338912:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762976.338915:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762976.338918:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762976.338919:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762976.338923:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762976.338924:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762976.338925:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762976.338926:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762976.338927:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762976.338928:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762976.338929:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762976.338930:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762976.338931:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762976.338932:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762976.338933:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762976.338934:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762976.338935:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762976.338936:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762976.338937:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762976.338938:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762976.338939:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762976.338940:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762976.338941:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762976.338942:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762976.338943:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762976.338944:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762976.338945:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762976.338946:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762976.338947:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762976.338948:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762976.338949:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762976.338950:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762976.338951:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762976.338953:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762976.338954:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762976.338955:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762976.338956:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762976.338957:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762976.338958:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762976.338959:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762976.338960:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762976.338961:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762976.338962:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762976.338964:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762976.338965:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762976.338966:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762976.338967:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762976.338968:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762976.338969:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762976.338970:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762976.338971:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762976.338972:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762976.338973:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762976.338974:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762976.338975:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762976.338976:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762976.338977:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762976.338978:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762976.338979:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762976.338980:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762976.338981:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762976.338982:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762976.338984:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762976.338985:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762976.338986:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762976.338987:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762976.338988:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762976.338989:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762976.338990:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762976.338991:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762976.338992:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762976.338993:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762976.338994:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762976.338995:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762976.338996:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762976.338997:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762976.338999:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762976.339000:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762976.339001:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762976.339002:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762976.339003:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762976.339004:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762976.339005:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762976.339006:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762976.339007:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762976.339008:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762976.339009:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762976.339010:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762976.339011:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762976.339012:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762976.339013:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762976.339014:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762976.339015:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762976.339016:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762976.339017:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762976.339018:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762976.339019:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762976.339021:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762976.339022:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762976.339023:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762976.339024:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762976.339025:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762976.339026:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762976.339027:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762976.339028:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762976.339029:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762976.339030:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762976.339031:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762976.339032:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762976.339033:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762976.339034:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762976.339035:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762976.339036:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762976.339037:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762976.339038:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762976.339039:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762976.339040:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762976.339041:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762976.339042:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762976.339043:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762976.339044:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762976.339045:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762976.339046:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762976.339047:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762976.339048:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762976.339049:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762976.339050:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762976.339051:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762976.339052:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762976.339054:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762976.339055:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762976.339056:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762976.339057:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762976.339058:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762976.339059:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762976.339060:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762976.339061:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762976.339062:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762976.339063:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762976.339064:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762976.339065:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762976.339066:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762976.339067:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762976.339068:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762976.339069:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762976.339070:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762976.339071:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762976.339073:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762976.339075:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762976.339076:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762976.339078:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762976.339080:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762976.339081:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762976.339083:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762976.339084:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762976.339086:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762976.339087:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762976.339089:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762976.339091:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762976.339093:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762976.339094:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762976.339097:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762976.339099:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762976.339101:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762976.339102:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762976.339104:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762976.339107:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762976.339109:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762976.339110:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762976.339112:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762976.339115:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762976.339117:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762976.339119:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762976.339122:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762976.339123:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762976.339126:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762976.339128:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762976.339130:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762976.339132:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762976.339134:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762976.339136:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762976.339138:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762976.339139:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762976.339140:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762976.339142:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762976.339143:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762976.339144:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762976.339145:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762976.339146:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762976.339147:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762976.339148:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762976.339149:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762976.339150:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762976.339152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762976.339153:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762976.339154:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762976.339156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762976.339157:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762976.339158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762976.339159:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762976.339160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762976.339161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762976.339162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762976.339163:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762976.339164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762976.339165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762976.339166:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762976.339167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762976.339168:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762976.339169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762976.339170:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762976.339171:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762976.339172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762976.339174:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762976.339175:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762976.339176:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762976.339177:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762976.339178:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762976.339179:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762976.339180:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762976.339187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762976.339189:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762976.339190:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762976.339191:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762976.339192:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762976.339194:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762976.339195:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762976.339196:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762976.339198:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762976.339199:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762976.339200:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762976.339201:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762976.339202:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762976.339203:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762976.339205:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762976.339206:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762976.339207:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762976.339208:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762976.339209:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762976.339210:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762976.339211:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762976.339295:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762976.339326:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762976.339350:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067576064:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762976.809916:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1789762977.338255:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [6400, 6655] 00000080:00200000:2.0:1789762977.338285:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762977.338292:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762977.338307:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 27262976 i_blocks 51207 00000020:00200000:2.0:1789762977.338311:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762977.338323:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762977.338563:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 27262976, count: 1048576 00000080:00200000:2.0:1789762977.338571:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 27262976, bytes: 1048576 00000080:00200000:2.0:1789762977.338581:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762977.338589:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762977.338591:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [27262976, 28311552) 0 0 00000080:00200000:2.0:1789762977.338595:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762977.338608:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [6656, 6911] 00020000:00200000:2.0:1789762977.338616:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 27262976 chunk: [27262976, 28311552) 28311552, 1048576 00020000:00200000:2.0:1789762977.338619:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762977.338637:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [6656, 6911] 00000008:00010000:2.0:1789762977.338651:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762977.338665:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762977.338681:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [27262976, 28311552) append 0 append lockpos 0 00000080:00200000:2.0:1789762977.338696:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=27262976=0x1a00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762977.338965:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762977.338969:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 28311552 00000020:00010000:2.0:1789762977.338978:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762977.338982:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762977.338986:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762977.338990:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762977.338994:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762977.339002:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762977.339008:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762977.339024:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762977.339308:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061de300 - object 0x280000400:15 offset 27262976<>28311552 00000100:00100000:2.0:1789762977.339348:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762977.339368:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067576832:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762977.538477:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968176 gen 4 00000008:00010000:2.0:1789762977.538543:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762977.538550:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762977.538553:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762977.538555:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762977.538556:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762977.538557:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762977.538558:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762977.538561:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762977.538562:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762977.538564:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762977.538565:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762977.538567:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762977.538576:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762977.538577:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762977.538579:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762977.538580:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762977.538581:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762977.538582:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762977.538584:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762977.538585:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762977.538586:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762977.538588:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762977.538589:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762977.538590:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762977.538591:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762977.538592:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762977.538593:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762977.538594:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762977.538595:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762977.538596:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762977.538598:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762977.538599:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762977.538600:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762977.538601:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762977.538602:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762977.538603:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762977.538604:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762977.538605:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762977.538607:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762977.538608:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762977.538610:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762977.538612:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762977.538613:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762977.538615:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762977.538616:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762977.538617:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762977.538618:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762977.538619:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762977.538620:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762977.538621:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762977.538622:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762977.538624:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762977.538625:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762977.538626:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762977.538627:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762977.538628:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762977.538629:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762977.538630:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762977.538631:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762977.538632:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762977.538633:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762977.538634:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762977.538635:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762977.538636:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762977.538637:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762977.538638:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762977.538639:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762977.538640:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762977.538641:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762977.538642:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762977.538643:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762977.538645:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762977.538646:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762977.538648:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762977.538649:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762977.538651:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762977.538652:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762977.538653:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762977.538654:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762977.538655:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762977.538656:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762977.538657:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762977.538658:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762977.538659:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762977.538660:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762977.538661:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762977.538662:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762977.538663:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762977.538664:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762977.538665:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762977.538666:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762977.538667:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762977.538674:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762977.538676:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762977.538677:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762977.538678:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762977.538679:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762977.538680:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762977.538681:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762977.538682:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762977.538683:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762977.538684:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762977.538685:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762977.538687:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762977.538688:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762977.538690:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762977.538691:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762977.538692:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762977.538693:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762977.538694:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762977.538696:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762977.538697:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762977.538698:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762977.538699:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762977.538700:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762977.538701:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762977.538702:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762977.538703:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762977.538704:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762977.538705:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762977.538706:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762977.538707:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762977.538708:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762977.538709:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762977.538710:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762977.538711:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762977.538712:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762977.538713:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762977.538714:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762977.538715:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762977.538716:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762977.538717:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762977.538718:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762977.538719:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762977.538720:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762977.538722:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762977.538723:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762977.538725:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762977.538726:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762977.538727:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762977.538728:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762977.538729:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762977.538730:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762977.538731:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762977.538732:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762977.538733:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762977.538734:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762977.538735:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762977.538737:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762977.538738:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762977.538739:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762977.538740:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762977.538741:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762977.538742:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762977.538743:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762977.538744:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762977.538745:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762977.538746:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762977.538747:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762977.538748:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762977.538749:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762977.538750:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762977.538751:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762977.538752:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762977.538753:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762977.538754:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762977.538755:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762977.538757:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762977.538758:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762977.538760:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762977.538761:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762977.538762:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762977.538763:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762977.538764:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762977.538765:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762977.538766:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762977.538767:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762977.538768:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762977.538769:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762977.538770:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762977.538771:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762977.538772:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762977.538773:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762977.538774:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762977.538775:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762977.538776:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762977.538777:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762977.538778:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762977.538779:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762977.538780:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762977.538781:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762977.538782:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762977.538783:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762977.538784:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762977.538785:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762977.538786:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762977.538787:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762977.538789:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762977.538790:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762977.538791:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762977.538793:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762977.538794:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762977.538795:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762977.538819:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762977.538820:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762977.538822:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762977.538823:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762977.538824:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762977.538826:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762977.538827:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762977.538828:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762977.538829:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762977.538830:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762977.538831:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762977.538832:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762977.538833:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762977.538834:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762977.538835:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762977.538836:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762977.538837:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762977.538838:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762977.538839:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762977.538840:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762977.538841:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762977.538842:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762977.538843:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762977.538844:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762977.538845:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762977.538846:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762977.538848:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762977.538849:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762977.538850:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762977.538852:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762977.538853:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762977.538854:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762977.538856:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762977.538857:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762977.538858:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762977.538859:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762977.538860:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762977.538861:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762977.538863:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762977.538864:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762977.538865:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762977.538866:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762977.538867:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762977.538868:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762977.538869:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762977.538870:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762977.538871:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762977.538872:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762977.538873:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762977.538874:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762977.538875:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762977.538877:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762977.538878:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762977.538879:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762977.538949:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762977.538967:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762977.538990:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067576832:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762977.643899:0:6823:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1789762977.643913:0:6823:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 2/0,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x420000010000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762977.643935:0:6823:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff9360421cb800 already unused, calling callback (ffffffffc1163300) 00010000:00010000:2.0:1789762977.643943:0:6823:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 3/0,0 mode: PR/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x428400010000 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762977.644007:0:6823:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 2/0,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762977.644015:0:6823:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1789762977.644045:0:6823:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 1/0,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762977.644073:0:6823:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-OST0000-osc-ffff936020592000 lock: 00000000a1031277/0xb758ac0edef9cb8f lrc: 0/0,0 mode: --/PR res: [0x280000401:0x9:0x0].0x0 rrc: 4 type: EXT [0->18446744073709551615] (req 0->1048575) gid 0 flags: 0x4c29400010020 nid: local remote: 0xdf5a7ac59f177ae3 expref: -99 pid: 16307 timeout: 0 lvb_type: 1 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789762977.644142:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762977.644146:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762977.644149:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00200000:3.0:1789762977.744737:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1789762977.744739:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1789762977.744740:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1789762977.744742:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1789762977.744743:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1789762977.744745:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1789762977.744746:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1789762977.744747:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1789762977.744748:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1789762977.744750:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1789762977.744751:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1789762977.744752:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1789762977.744753:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1789762977.744754:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1789762977.744756:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1789762977.744757:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1789762977.744758:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1789762977.744759:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1789762977.744761:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1789762977.744762:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1789762977.744763:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1789762977.744764:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1789762977.744766:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1789762977.744767:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1789762977.744768:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1789762977.744769:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1789762977.744771:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1789762977.744772:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1789762977.744773:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1789762977.744774:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1789762977.744775:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1789762977.744776:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1789762977.744778:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1789762977.744779:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1789762977.744781:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1789762977.744783:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1789762977.744784:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1789762977.744785:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1789762977.744787:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1789762977.744788:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1789762977.744789:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1789762977.744791:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1789762977.744792:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1789762977.744793:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1789762977.744794:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1789762977.744818:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1789762977.744819:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1789762977.744821:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1789762977.744822:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1789762977.744823:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1789762977.744824:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1789762977.744826:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1789762977.744827:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1789762977.744828:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1789762977.744829:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1789762977.744830:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1789762977.744832:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1789762977.744833:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1789762977.744834:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1789762977.744835:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1789762977.744836:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1789762977.744838:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1789762977.744839:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1789762977.744840:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1789762977.744841:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1789762977.744843:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1789762977.744844:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1789762977.744846:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1789762977.744847:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1789762977.744849:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1789762977.744850:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1789762977.744851:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1789762977.744852:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1789762977.744854:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1789762977.744855:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1789762977.744856:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1789762977.744857:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1789762977.744859:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1789762977.744860:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1789762977.744861:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1789762977.744862:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1789762977.744864:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1789762977.744865:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1789762977.744866:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1789762977.744867:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1789762977.744868:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1789762977.744870:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1789762977.744871:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1789762977.744872:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1789762977.744873:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1789762977.744875:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1789762977.744876:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1789762977.744877:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1789762977.744878:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1789762977.744880:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1789762977.744881:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1789762977.744882:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1789762977.744884:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1789762977.744885:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1789762977.744887:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1789762977.744888:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1789762977.744889:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1789762977.744891:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1789762977.744892:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1789762977.744893:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1789762977.744895:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1789762977.744896:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1789762977.744897:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1789762977.744898:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1789762977.744899:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1789762977.744901:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1789762977.744902:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1789762977.744903:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1789762977.744904:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1789762977.744905:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1789762977.744907:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1789762977.744908:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1789762977.744909:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1789762977.744910:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1789762977.744911:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1789762977.744913:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1789762977.744914:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1789762977.744916:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1789762977.744917:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1789762977.744990:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1789762977.745005:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdc600 00000100:00100000:3.0:1789762977.745029:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067576960:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762977.745072:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [6912, 7167] 00000080:00200000:2.0:1789762977.745093:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762977.745099:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762977.745113:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 29360128 i_blocks 55303 00000020:00200000:2.0:1789762977.745116:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762977.745131:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762977.745381:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 29360128, count: 1048576 00000080:00200000:2.0:1789762977.745389:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 29360128, bytes: 1048576 00000080:00200000:2.0:1789762977.745399:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762977.745407:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762977.745410:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [29360128, 30408704) 0 0 00000080:00200000:2.0:1789762977.745413:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762977.745428:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [7168, 7423] 00020000:00200000:2.0:1789762977.745435:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 29360128 chunk: [29360128, 30408704) 30408704, 1048576 00020000:00200000:2.0:1789762977.745439:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762977.745457:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [7168, 7423] 00000008:00010000:2.0:1789762977.745471:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762977.745476:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762977.745487:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [29360128, 30408704) append 0 append lockpos 0 00000080:00200000:2.0:1789762977.745498:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=29360128=0x1c00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762977.745732:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762977.745735:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 30408704 00000020:00010000:2.0:1789762977.745741:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762977.745744:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762977.745747:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762977.745749:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762977.745751:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762977.745758:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762977.745763:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762977.745779:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762977.746060:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 29360128<>30408704 00000100:00100000:2.0:1789762977.746102:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577344:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762977.834015:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762977.942779:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968179 gen 4 00000008:00010000:2.0:1789762977.942870:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762977.942878:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762977.942881:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762977.942882:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762977.942883:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762977.942885:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762977.942887:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762977.942888:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762977.942889:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762977.942891:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762977.942892:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762977.942893:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762977.942894:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762977.942896:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762977.942897:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762977.942898:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762977.942899:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762977.942900:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762977.942901:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762977.942902:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762977.942903:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762977.942904:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762977.942905:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762977.942906:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762977.942907:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762977.942908:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762977.942909:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762977.942910:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762977.942911:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762977.942912:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762977.942913:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762977.942914:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762977.942915:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762977.942916:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762977.942917:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762977.942918:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762977.942919:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762977.942921:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762977.942922:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762977.942924:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762977.942925:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762977.942926:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762977.942928:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762977.942929:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762977.942930:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762977.942931:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762977.942932:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762977.942933:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762977.942934:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762977.942935:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762977.942936:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762977.942937:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762977.942938:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762977.942939:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762977.942940:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762977.942941:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762977.942942:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762977.942943:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762977.942944:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762977.942945:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762977.942946:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762977.942947:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762977.942948:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762977.942949:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762977.942950:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762977.942951:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762977.942952:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762977.942953:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762977.942954:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762977.942956:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762977.942958:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762977.942959:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762977.942960:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762977.942961:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762977.942963:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762977.942964:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762977.942965:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762977.942966:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762977.942967:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762977.942968:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762977.942969:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762977.942970:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762977.942971:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762977.942972:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762977.942974:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762977.942975:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762977.942976:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762977.942977:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762977.942978:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762977.942979:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762977.942980:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762977.942981:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762977.942982:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762977.942983:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762977.942984:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762977.942985:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762977.942987:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762977.942988:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762977.942989:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762977.942990:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762977.942991:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762977.942992:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762977.942994:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762977.942995:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762977.942997:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762977.942998:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762977.942999:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762977.943000:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762977.943001:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762977.943002:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762977.943003:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762977.943005:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762977.943006:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762977.943007:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762977.943008:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762977.943009:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762977.943010:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762977.943011:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762977.943012:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762977.943013:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762977.943014:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762977.943015:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762977.943016:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762977.943017:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762977.943019:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762977.943020:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762977.943022:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762977.943023:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762977.943025:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762977.943027:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762977.943028:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762977.943030:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762977.943032:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762977.943034:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762977.943036:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762977.943037:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762977.943039:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762977.943041:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762977.943043:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762977.943045:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762977.943047:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762977.943049:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762977.943051:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762977.943053:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762977.943055:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762977.943058:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762977.943060:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762977.943063:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762977.943065:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762977.943067:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762977.943069:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762977.943071:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762977.943073:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762977.943075:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762977.943076:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762977.943078:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762977.943079:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762977.943080:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762977.943082:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762977.943083:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762977.943084:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762977.943085:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762977.943086:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762977.943087:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762977.943088:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762977.943090:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762977.943091:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762977.943093:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762977.943094:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762977.943095:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762977.943096:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762977.943097:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762977.943098:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762977.943100:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762977.943101:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762977.943102:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762977.943103:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762977.943104:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762977.943105:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762977.943106:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762977.943107:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762977.943108:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762977.943109:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762977.943110:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762977.943111:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762977.943112:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762977.943114:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762977.943115:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762977.943116:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762977.943117:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762977.943118:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762977.943119:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762977.943120:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762977.943121:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762977.943122:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762977.943123:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762977.943124:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762977.943126:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762977.943128:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762977.943129:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762977.943130:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762977.943132:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762977.943133:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762977.943134:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762977.943135:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762977.943136:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762977.943137:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762977.943138:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762977.943139:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762977.943140:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762977.943141:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762977.943142:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762977.943143:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762977.943144:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762977.943145:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762977.943146:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762977.943147:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762977.943149:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762977.943150:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762977.943151:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762977.943152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762977.943153:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762977.943154:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762977.943155:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762977.943156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762977.943157:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762977.943158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762977.943160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762977.943161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762977.943162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762977.943164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762977.943165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762977.943167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762977.943168:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762977.943169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762977.943170:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762977.943172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762977.943173:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762977.943174:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762977.943175:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762977.943176:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762977.943177:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762977.943178:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762977.943179:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762977.943180:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762977.943181:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762977.943182:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762977.943183:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762977.943184:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762977.943185:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762977.943186:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762977.943187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762977.943193:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762977.943194:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762977.943195:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762977.943197:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762977.943198:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762977.943287:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762977.943306:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762977.943341:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577344:192.168.203.137@tcp:4:dd.500 00000080:00200000:3.0:1789762977.943384:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [7168, 7423] 00000080:00200000:3.0:1789762977.943405:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1789762977.943411:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1789762977.943428:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 30408704 i_blocks 57351 00000020:00200000:3.0:1789762977.943432:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1789762977.943443:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1789762977.943688:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 30408704, count: 1048576 00000080:00200000:3.0:1789762977.943698:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 30408704, bytes: 1048576 00000080:00200000:3.0:1789762977.943714:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:3.0:1789762977.943723:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:3.0:1789762977.943725:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [30408704, 31457280) 0 0 00000080:00200000:3.0:1789762977.943730:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1789762977.943747:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [7424, 7679] 00020000:00200000:3.0:1789762977.943755:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 30408704 chunk: [30408704, 31457280) 31457280, 1048576 00020000:00200000:3.0:1789762977.943760:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1789762977.943779:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [7424, 7679] 00000008:00010000:3.0:1789762977.943833:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1789762977.943840:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9360061427b0, osc lock ffff936003e85300, flags 0x0 00000080:00200000:3.0:1789762977.943851:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [30408704, 31457280) append 0 append lockpos 0 00000080:00200000:3.0:1789762977.943866:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=30408704=0x1d00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:3.0:1789762977.944136:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1789762977.944140:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 31457280 00000020:00010000:3.0:1789762977.944147:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:3.0:1789762977.944150:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:3.0:1789762977.944153:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061427b0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1789762977.944156:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:3.0:1789762977.944158:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061427b0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:3.0:1789762977.944164:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:3.0:1789762977.944169:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1789762977.944201:0:419:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff936010cdc600 00000008:00100000:3.0:1789762977.944533:0:419:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff936010aa4000 - object 0x280000400:15 offset 30408704<>31457280 00000100:00100000:2.0:1789762977.944594:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762977.944611:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577472:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762978.147415:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968180 gen 4 00000008:00010000:2.0:1789762978.147476:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdc600 00000020:00200000:2.0:1789762978.147484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762978.147487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762978.147488:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762978.147489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762978.147491:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762978.147492:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762978.147493:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762978.147494:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762978.147496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762978.147498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762978.147499:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762978.147500:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762978.147501:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762978.147502:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762978.147503:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762978.147505:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762978.147507:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762978.147508:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762978.147509:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762978.147511:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762978.147513:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762978.147515:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762978.147516:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762978.147520:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762978.147522:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762978.147523:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762978.147524:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762978.147525:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762978.147527:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762978.147528:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762978.147530:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762978.147531:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762978.147532:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762978.147534:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762978.147535:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762978.147536:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762978.147537:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762978.147538:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762978.147539:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762978.147540:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762978.147541:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762978.147542:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762978.147543:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762978.147544:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762978.147545:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762978.147546:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762978.147547:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762978.147548:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762978.147550:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762978.147551:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762978.147552:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762978.147553:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762978.147554:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762978.147554:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762978.147556:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762978.147557:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762978.147558:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762978.147559:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762978.147560:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762978.147561:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762978.147563:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762978.147564:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762978.147566:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762978.147567:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762978.147568:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762978.147569:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762978.147570:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762978.147571:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762978.147572:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762978.147573:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762978.147574:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762978.147575:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762978.147576:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762978.147577:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762978.147578:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762978.147580:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762978.147581:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762978.147582:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762978.147583:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762978.147584:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762978.147585:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762978.147586:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762978.147588:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762978.147589:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762978.147590:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762978.147591:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762978.147592:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762978.147593:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762978.147594:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762978.147595:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762978.147596:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762978.147598:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762978.147599:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762978.147601:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762978.147602:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762978.147604:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762978.147605:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762978.147606:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762978.147607:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762978.147608:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762978.147609:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762978.147610:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762978.147611:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762978.147612:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762978.147613:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762978.147614:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762978.147615:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762978.147616:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762978.147617:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762978.147618:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762978.147620:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762978.147621:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762978.147622:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762978.147623:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762978.147624:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762978.147625:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762978.147626:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762978.147628:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762978.147629:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762978.147630:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762978.147631:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762978.147632:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762978.147633:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762978.147634:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762978.147635:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762978.147636:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762978.147638:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762978.147639:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762978.147640:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762978.147642:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762978.147643:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762978.147644:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762978.147645:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762978.147646:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762978.147647:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762978.147648:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762978.147649:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762978.147650:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762978.147651:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762978.147653:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762978.147654:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762978.147656:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762978.147658:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762978.147659:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762978.147661:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762978.147662:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762978.147664:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762978.147665:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762978.147667:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762978.147669:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762978.147670:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762978.147672:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762978.147674:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762978.147676:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762978.147678:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762978.147679:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762978.147682:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762978.147684:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762978.147687:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762978.147689:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762978.147691:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762978.147694:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762978.147696:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762978.147698:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762978.147700:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762978.147702:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762978.147705:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762978.147707:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762978.147709:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762978.147711:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762978.147712:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762978.147714:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762978.147715:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762978.147716:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762978.147718:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762978.147719:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762978.147720:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762978.147721:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762978.147722:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762978.147723:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762978.147724:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762978.147725:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762978.147726:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762978.147727:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762978.147728:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762978.147729:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762978.147730:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762978.147731:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762978.147733:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762978.147734:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762978.147736:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762978.147737:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762978.147738:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762978.147739:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762978.147740:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762978.147741:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762978.147742:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762978.147743:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762978.147744:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762978.147745:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762978.147747:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762978.147748:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762978.147749:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762978.147750:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762978.147751:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762978.147752:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762978.147753:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762978.147754:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762978.147755:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762978.147756:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762978.147757:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762978.147758:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762978.147759:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762978.147760:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762978.147761:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762978.147762:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762978.147763:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762978.147764:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762978.147765:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762978.147766:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762978.147768:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762978.147769:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762978.147770:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762978.147771:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762978.147773:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762978.147774:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762978.147775:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762978.147776:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762978.147777:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762978.147778:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762978.147779:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762978.147780:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762978.147781:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762978.147782:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762978.147783:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762978.147785:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762978.147786:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762978.147787:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762978.147788:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762978.147789:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762978.147790:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762978.147791:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762978.147792:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762978.147793:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762978.147794:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762978.147795:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762978.147796:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762978.147842:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762978.147844:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762978.147846:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762978.147848:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762978.147850:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762978.147852:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762978.147855:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762978.147857:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762978.147859:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762978.148010:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762978.148032:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdc600 00000100:00100000:2.0:1789762978.148058:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577472:192.168.203.137@tcp:4:dd.500 00000080:00200000:3.0:1789762978.148127:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [7424, 7679] 00000080:00200000:3.0:1789762978.148151:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1789762978.148159:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1789762978.148177:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 31457280 i_blocks 59399 00000020:00200000:3.0:1789762978.148182:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1789762978.148198:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1789762978.148568:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 31457280, count: 1048576 00000080:00200000:3.0:1789762978.148579:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 31457280, bytes: 1048576 00000080:00200000:3.0:1789762978.148590:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:3.0:1789762978.148600:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:3.0:1789762978.148602:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [31457280, 32505856) 0 0 00000080:00200000:3.0:1789762978.148608:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1789762978.148625:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [7680, 7935] 00020000:00200000:3.0:1789762978.148633:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 31457280 chunk: [31457280, 32505856) 32505856, 1048576 00020000:00200000:3.0:1789762978.148637:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1789762978.148658:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [7680, 7935] 00000008:00010000:3.0:1789762978.148673:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1789762978.148679:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9360061427b0, osc lock ffff936003e85300, flags 0x0 00000080:00200000:3.0:1789762978.148691:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [31457280, 32505856) append 0 append lockpos 0 00000080:00200000:3.0:1789762978.148707:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=31457280=0x1e00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:3.0:1789762978.149101:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1789762978.149106:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 32505856 00000020:00010000:3.0:1789762978.149114:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:3.0:1789762978.149118:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.149122:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061427b0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.149125:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:3.0:1789762978.149129:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061427b0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:3.0:1789762978.149136:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:3.0:1789762978.149143:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1789762978.149163:0:419:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff936010cdce40 00000008:00100000:3.0:1789762978.149570:0:419:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff936010aa4700 - object 0x280000400:15 offset 31457280<>32505856 00000100:00100000:3.0:1789762978.149622:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1789762978.149645:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067577600:192.168.203.137@tcp:4:dd.500 00000100:00100000:3.0:1789762978.342954:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968181 gen 4 00000008:00010000:3.0:1789762978.343035:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdce40 00000020:00200000:3.0:1789762978.343044:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1789762978.343048:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1789762978.343050:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1789762978.343053:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1789762978.343055:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1789762978.343057:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1789762978.343059:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1789762978.343061:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1789762978.343063:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1789762978.343065:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1789762978.343067:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1789762978.343069:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1789762978.343071:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1789762978.343073:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1789762978.343075:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1789762978.343077:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1789762978.343079:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1789762978.343081:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1789762978.343083:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1789762978.343085:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1789762978.343088:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1789762978.343090:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1789762978.343092:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1789762978.343094:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1789762978.343096:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1789762978.343098:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1789762978.343121:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1789762978.343122:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1789762978.343124:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1789762978.343125:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1789762978.343127:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1789762978.343128:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1789762978.343130:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1789762978.343131:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1789762978.343132:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1789762978.343134:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1789762978.343135:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1789762978.343136:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1789762978.343138:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1789762978.343139:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1789762978.343140:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1789762978.343142:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1789762978.343143:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1789762978.343144:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1789762978.343146:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1789762978.343147:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1789762978.343148:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1789762978.343150:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1789762978.343151:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1789762978.343152:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1789762978.343153:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1789762978.343155:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1789762978.343156:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1789762978.343157:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1789762978.343158:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1789762978.343160:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1789762978.343161:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1789762978.343163:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1789762978.343164:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1789762978.343165:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1789762978.343167:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1789762978.343186:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1789762978.343189:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1789762978.343191:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1789762978.343193:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1789762978.343195:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1789762978.343197:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1789762978.343199:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1789762978.343201:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1789762978.343203:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1789762978.343205:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1789762978.343206:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1789762978.343208:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1789762978.343210:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1789762978.343212:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1789762978.343214:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1789762978.343216:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1789762978.343218:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1789762978.343220:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1789762978.343222:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1789762978.343224:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1789762978.343226:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1789762978.343227:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1789762978.343229:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1789762978.343231:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1789762978.343233:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1789762978.343235:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1789762978.343237:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1789762978.343239:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1789762978.343241:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1789762978.343243:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1789762978.343245:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1789762978.343247:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1789762978.343249:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1789762978.343251:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1789762978.343253:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1789762978.343255:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1789762978.343257:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1789762978.343279:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1789762978.343283:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1789762978.343285:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1789762978.343287:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1789762978.343288:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1789762978.343290:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1789762978.343292:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1789762978.343294:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1789762978.343296:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1789762978.343304:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1789762978.343306:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1789762978.343308:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1789762978.343311:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1789762978.343313:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1789762978.343315:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1789762978.343317:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1789762978.343319:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1789762978.343321:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1789762978.343323:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1789762978.343325:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1789762978.343326:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1789762978.343328:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1789762978.343330:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1789762978.343332:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1789762978.343334:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1789762978.343336:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1789762978.343339:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1789762978.343341:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1789762978.343343:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1789762978.343345:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1789762978.343347:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1789762978.343349:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1789762978.343351:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1789762978.343353:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1789762978.343355:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1789762978.343357:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1789762978.343359:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1789762978.343360:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1789762978.343362:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1789762978.343364:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1789762978.343366:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1789762978.343368:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1789762978.343370:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1789762978.343372:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1789762978.343373:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1789762978.343375:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1789762978.343377:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1789762978.343379:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1789762978.343381:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1789762978.343383:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1789762978.343385:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1789762978.343387:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1789762978.343388:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1789762978.343390:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1789762978.343409:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1789762978.343411:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1789762978.343413:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1789762978.343415:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1789762978.343418:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1789762978.343421:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1789762978.343423:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1789762978.343425:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1789762978.343428:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1789762978.343430:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1789762978.343432:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1789762978.343434:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1789762978.343436:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1789762978.343439:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1789762978.343440:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1789762978.343442:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1789762978.343445:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1789762978.343447:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1789762978.343449:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1789762978.343451:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1789762978.343453:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1789762978.343455:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1789762978.343457:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1789762978.343459:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1789762978.343461:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1789762978.343463:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1789762978.343465:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1789762978.343466:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1789762978.343469:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1789762978.343471:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1789762978.343473:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1789762978.343475:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1789762978.343495:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1789762978.343497:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1789762978.343499:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1789762978.343501:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1789762978.343504:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1789762978.343506:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1789762978.343508:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1789762978.343510:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1789762978.343512:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1789762978.343514:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1789762978.343516:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1789762978.343518:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1789762978.343520:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1789762978.343522:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1789762978.343524:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1789762978.343526:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1789762978.343528:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1789762978.343530:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1789762978.343532:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1789762978.343534:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1789762978.343535:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1789762978.343537:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1789762978.343539:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1789762978.343541:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1789762978.343543:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1789762978.343545:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1789762978.343546:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1789762978.343548:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1789762978.343550:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1789762978.343552:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1789762978.343554:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1789762978.343556:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1789762978.343557:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1789762978.343559:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1789762978.343561:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1789762978.343563:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1789762978.343566:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1789762978.343568:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1789762978.343570:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1789762978.343573:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1789762978.343575:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1789762978.343577:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1789762978.343578:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1789762978.343580:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1789762978.343582:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1789762978.343584:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1789762978.343586:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1789762978.343588:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1789762978.343589:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1789762978.343591:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1789762978.343593:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1789762978.343595:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1789762978.343597:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1789762978.343599:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1789762978.343601:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1789762978.343603:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1789762978.343604:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1789762978.343606:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1789762978.343608:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1789762978.343610:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1789762978.343612:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1789762978.343614:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1789762978.343616:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1789762978.343618:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1789762978.343620:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1789762978.343622:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1789762978.343623:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1789762978.343625:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1789762978.343628:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1789762978.343630:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1789762978.343633:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1789762978.343635:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1789762978.343745:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1789762978.343775:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdce40 00000080:00200000:2.0:1789762978.343837:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [7680, 7935] 00000080:00200000:2.0:1789762978.343859:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:3.0:1789762978.343870:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067577600:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762978.343871:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762978.343901:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 32505856 i_blocks 61447 00000020:00200000:2.0:1789762978.343910:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762978.343933:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762978.344218:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 32505856, count: 1048576 00000080:00200000:2.0:1789762978.344227:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 32505856, bytes: 1048576 00000080:00200000:2.0:1789762978.344238:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762978.344246:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762978.344249:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [32505856, 33554432) 0 0 00000080:00200000:2.0:1789762978.344253:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762978.344440:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [7936, 8191] 00020000:00200000:2.0:1789762978.344448:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 32505856 chunk: [32505856, 33554432) 33554432, 1048576 00020000:00200000:2.0:1789762978.344452:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762978.344483:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [7936, 8191] 00000008:00010000:2.0:1789762978.344498:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762978.344503:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762978.344514:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [32505856, 33554432) append 0 append lockpos 0 00000080:00200000:2.0:1789762978.344526:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=32505856=0x1f00000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762978.344831:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762978.344833:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 33554432 00000020:00010000:2.0:1789762978.344841:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762978.344844:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762978.344847:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762978.344851:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762978.344853:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762978.344860:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762978.344866:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762978.344882:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762978.345142:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 32505856<>33554432 00000100:00100000:2.0:1789762978.345180:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762978.345197:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577728:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762978.541057:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968182 gen 4 00000008:00010000:2.0:1789762978.541118:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762978.541125:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762978.541128:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762978.541129:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762978.541131:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762978.541132:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762978.541133:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762978.541134:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762978.541135:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762978.541136:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762978.541137:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762978.541138:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762978.541139:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762978.541140:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762978.541141:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762978.541142:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762978.541143:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762978.541144:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762978.541145:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762978.541147:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762978.541149:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762978.541150:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762978.541152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762978.541153:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762978.541154:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762978.541155:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762978.541156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762978.541157:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762978.541158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762978.541159:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762978.541160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762978.541161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762978.541162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762978.541164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762978.541164:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762978.541165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762978.541166:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762978.541167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762978.541169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762978.541170:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762978.541171:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762978.541172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762978.541174:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762978.541176:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762978.541177:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762978.541179:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762978.541180:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762978.541182:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762978.541184:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762978.541186:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762978.541187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762978.541189:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762978.541191:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762978.541193:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762978.541204:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762978.541207:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762978.541209:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762978.541210:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762978.541212:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762978.541214:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762978.541216:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762978.541218:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762978.541220:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762978.541222:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762978.541224:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762978.541226:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762978.541228:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762978.541230:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762978.541232:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762978.541235:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762978.541237:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762978.541239:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762978.541241:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762978.541243:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762978.541245:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762978.541246:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762978.541248:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762978.541249:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762978.541250:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762978.541251:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762978.541252:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762978.541253:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762978.541254:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762978.541255:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762978.541270:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762978.541273:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762978.541275:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762978.541277:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762978.541279:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762978.541281:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762978.541283:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762978.541285:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762978.541287:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762978.541289:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762978.541291:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762978.541293:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762978.541295:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762978.541297:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762978.541299:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762978.541301:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762978.541303:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762978.541305:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762978.541308:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762978.541310:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762978.541312:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762978.541314:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762978.541316:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762978.541318:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762978.541320:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762978.541322:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762978.541323:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762978.541324:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762978.541325:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762978.541326:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762978.541328:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762978.541329:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762978.541331:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762978.541332:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762978.541334:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762978.541335:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762978.541336:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762978.541337:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762978.541338:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762978.541340:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762978.541341:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762978.541342:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762978.541343:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762978.541344:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762978.541345:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762978.541346:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762978.541347:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762978.541348:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762978.541349:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762978.541350:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762978.541351:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762978.541352:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762978.541353:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762978.541354:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762978.541355:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762978.541356:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762978.541357:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762978.541358:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762978.541359:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762978.541360:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762978.541361:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762978.541362:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762978.541370:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762978.541372:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762978.541374:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762978.541377:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762978.541379:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762978.541381:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762978.541382:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762978.541383:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762978.541384:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762978.541385:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762978.541386:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762978.541387:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762978.541388:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762978.541389:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762978.541390:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762978.541392:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762978.541393:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762978.541394:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762978.541395:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762978.541396:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762978.541397:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762978.541398:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762978.541399:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762978.541400:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762978.541402:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762978.541403:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762978.541404:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762978.541405:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762978.541406:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762978.541407:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762978.541409:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762978.541410:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762978.541412:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762978.541413:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762978.541414:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762978.541416:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762978.541417:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762978.541419:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762978.541420:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762978.541421:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762978.541422:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762978.541423:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762978.541424:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762978.541425:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762978.541426:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762978.541427:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762978.541428:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762978.541429:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762978.541430:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762978.541431:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762978.541432:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762978.541433:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762978.541434:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762978.541435:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762978.541436:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762978.541437:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762978.541438:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762978.541439:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762978.541440:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762978.541441:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762978.541442:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762978.541443:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762978.541444:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762978.541445:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762978.541446:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762978.541447:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762978.541449:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762978.541450:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762978.541452:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762978.541453:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762978.541454:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762978.541455:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762978.541456:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762978.541457:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762978.541458:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762978.541460:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762978.541461:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762978.541462:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762978.541463:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762978.541464:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762978.541465:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762978.541466:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762978.541467:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762978.541468:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762978.541469:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762978.541470:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762978.541471:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762978.541472:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762978.541473:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762978.541474:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762978.541475:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762978.541476:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762978.541477:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762978.541478:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762978.541479:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762978.541480:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762978.541481:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762978.541483:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762978.541484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762978.541486:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762978.541487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762978.541488:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762978.541489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762978.541491:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762978.541492:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762978.541493:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762978.541494:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762978.541495:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762978.541496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762978.541497:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762978.541498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762978.541565:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762978.541583:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762978.541611:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577728:192.168.203.137@tcp:4:dd.500 00000080:00200000:3.0:1789762978.541617:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [7936, 8191] 00000080:00200000:3.0:1789762978.541637:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1789762978.541642:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1789762978.541657:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 33554432 i_blocks 63495 00000020:00200000:3.0:1789762978.541661:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1789762978.541674:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1789762978.541943:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 33554432, count: 1048576 00000080:00200000:3.0:1789762978.541950:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 33554432, bytes: 1048576 00000080:00200000:3.0:1789762978.541958:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:3.0:1789762978.541965:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:3.0:1789762978.541967:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [33554432, 34603008) 0 0 00000080:00200000:3.0:1789762978.541970:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1789762978.541983:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [8192, 8447] 00020000:00200000:3.0:1789762978.541989:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 33554432 chunk: [33554432, 34603008) 34603008, 1048576 00020000:00200000:3.0:1789762978.541991:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1789762978.542005:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [8192, 8447] 00000008:00010000:3.0:1789762978.542018:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1789762978.542022:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9360061427b0, osc lock ffff936003e85300, flags 0x0 00000080:00200000:3.0:1789762978.542031:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [33554432, 34603008) append 0 append lockpos 0 00000080:00200000:3.0:1789762978.542042:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=33554432=0x2000000, pages 256 (max 131072), lockless, parallel 00000020:00200000:3.0:1789762978.542241:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1789762978.542244:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 34603008 00000020:00010000:3.0:1789762978.542249:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:3.0:1789762978.542251:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.542254:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061427b0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.542257:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:3.0:1789762978.542277:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061427b0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:3.0:1789762978.542284:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:3.0:1789762978.542288:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1789762978.542309:0:419:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff936010cdce40 00000008:00100000:3.0:1789762978.542551:0:419:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff936010aa4700 - object 0x280000400:15 offset 33554432<>34603008 00000100:00100000:3.0:1789762978.542587:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1789762978.542605:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067577856:192.168.203.137@tcp:4:dd.500 00000100:00100000:3.0:1789762978.739750:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968183 gen 4 00000008:00010000:3.0:1789762978.739866:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdce40 00000020:00200000:3.0:1789762978.739874:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1789762978.739876:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1789762978.739878:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1789762978.739879:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1789762978.739881:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1789762978.739882:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1789762978.739883:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1789762978.739884:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1789762978.739885:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1789762978.739886:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1789762978.739887:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1789762978.739888:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1789762978.739890:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1789762978.739891:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1789762978.739892:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1789762978.739893:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1789762978.739894:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1789762978.739895:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1789762978.739896:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1789762978.739898:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1789762978.739899:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1789762978.739901:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1789762978.739902:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1789762978.739904:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1789762978.739905:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1789762978.739906:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1789762978.739907:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1789762978.739908:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1789762978.739916:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1789762978.739917:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1789762978.739919:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1789762978.739920:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1789762978.739921:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1789762978.739922:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1789762978.739923:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1789762978.739924:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1789762978.739925:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1789762978.739927:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1789762978.739928:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1789762978.739930:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1789762978.739931:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1789762978.739932:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1789762978.739933:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1789762978.739934:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1789762978.739935:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1789762978.739936:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1789762978.739937:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1789762978.739938:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1789762978.739939:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1789762978.739940:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1789762978.739942:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1789762978.739943:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1789762978.739945:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1789762978.739946:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1789762978.739947:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1789762978.739949:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1789762978.739950:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1789762978.739951:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1789762978.739952:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1789762978.739953:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1789762978.739954:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1789762978.739955:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1789762978.739956:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1789762978.739957:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1789762978.739958:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1789762978.739959:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1789762978.739960:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1789762978.739961:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1789762978.739962:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1789762978.739963:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1789762978.739964:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1789762978.739965:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1789762978.739966:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1789762978.739967:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1789762978.739968:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1789762978.739969:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1789762978.739970:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1789762978.739971:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1789762978.739972:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1789762978.739973:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1789762978.739974:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1789762978.739975:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1789762978.739976:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1789762978.739978:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1789762978.739979:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1789762978.739981:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1789762978.739982:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1789762978.739983:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1789762978.739984:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1789762978.739985:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1789762978.739986:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1789762978.739987:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1789762978.739988:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1789762978.739989:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1789762978.739991:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1789762978.739992:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1789762978.739993:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1789762978.739994:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1789762978.739995:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1789762978.739996:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1789762978.739997:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1789762978.739998:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1789762978.739999:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1789762978.740000:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1789762978.740001:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1789762978.740002:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1789762978.740003:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1789762978.740004:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1789762978.740005:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1789762978.740006:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1789762978.740007:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1789762978.740008:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1789762978.740009:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1789762978.740010:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1789762978.740011:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1789762978.740013:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1789762978.740015:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1789762978.740016:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1789762978.740017:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1789762978.740019:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1789762978.740020:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1789762978.740021:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1789762978.740022:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1789762978.740023:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1789762978.740024:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1789762978.740025:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1789762978.740026:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1789762978.740027:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1789762978.740028:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1789762978.740029:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1789762978.740030:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1789762978.740031:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1789762978.740032:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1789762978.740033:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1789762978.740034:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1789762978.740035:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1789762978.740036:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1789762978.740037:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1789762978.740038:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1789762978.740039:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1789762978.740040:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1789762978.740041:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1789762978.740042:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1789762978.740043:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1789762978.740044:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1789762978.740046:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1789762978.740047:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1789762978.740048:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1789762978.740050:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1789762978.740051:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1789762978.740053:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1789762978.740054:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1789762978.740055:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1789762978.740056:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1789762978.740057:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1789762978.740058:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1789762978.740059:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1789762978.740060:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1789762978.740061:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1789762978.740062:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1789762978.740063:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1789762978.740064:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1789762978.740065:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1789762978.740067:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1789762978.740068:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1789762978.740069:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1789762978.740070:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1789762978.740071:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1789762978.740072:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1789762978.740073:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1789762978.740074:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1789762978.740075:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1789762978.740076:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1789762978.740077:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1789762978.740078:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1789762978.740079:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1789762978.740080:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1789762978.740081:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1789762978.740082:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1789762978.740084:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1789762978.740085:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1789762978.740086:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1789762978.740087:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1789762978.740089:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1789762978.740090:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1789762978.740091:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1789762978.740092:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1789762978.740093:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1789762978.740094:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1789762978.740095:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1789762978.740096:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1789762978.740097:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1789762978.740098:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1789762978.740099:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1789762978.740100:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1789762978.740101:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1789762978.740102:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1789762978.740103:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1789762978.740104:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1789762978.740106:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1789762978.740107:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1789762978.740108:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1789762978.740109:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1789762978.740110:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1789762978.740111:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1789762978.740111:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1789762978.740112:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1789762978.740113:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1789762978.740114:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1789762978.740115:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1789762978.740116:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1789762978.740118:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1789762978.740119:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1789762978.740121:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1789762978.740122:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1789762978.740123:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1789762978.740124:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1789762978.740125:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1789762978.740126:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1789762978.740127:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1789762978.740128:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1789762978.740129:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1789762978.740130:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1789762978.740131:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1789762978.740132:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1789762978.740133:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1789762978.740134:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1789762978.740135:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1789762978.740136:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1789762978.740137:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1789762978.740139:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1789762978.740140:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1789762978.740141:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1789762978.740142:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1789762978.740143:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1789762978.740144:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1789762978.740145:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1789762978.740146:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1789762978.740147:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1789762978.740148:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1789762978.740149:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1789762978.740150:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1789762978.740151:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1789762978.740153:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1789762978.740154:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1789762978.740155:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1789762978.740157:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1789762978.740158:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1789762978.740159:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1789762978.740160:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1789762978.740161:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1789762978.740162:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1789762978.740163:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1789762978.740164:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1789762978.740165:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1789762978.740166:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1789762978.740231:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1789762978.740245:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdce40 00000100:00100000:3.0:1789762978.740286:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067577856:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762978.740297:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [8192, 8447] 00000080:00200000:2.0:1789762978.740313:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762978.740319:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762978.740334:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 34603008 i_blocks 65543 00000020:00200000:2.0:1789762978.740338:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762978.740352:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762978.740600:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 34603008, count: 1048576 00000080:00200000:2.0:1789762978.740608:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 34603008, bytes: 1048576 00000080:00200000:2.0:1789762978.740617:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762978.740626:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762978.740629:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [34603008, 35651584) 0 0 00000080:00200000:2.0:1789762978.740634:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762978.740648:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [8448, 8703] 00020000:00200000:2.0:1789762978.740656:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 34603008 chunk: [34603008, 35651584) 35651584, 1048576 00020000:00200000:2.0:1789762978.740660:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762978.740678:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [8448, 8703] 00000008:00010000:2.0:1789762978.740694:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762978.740699:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762978.740709:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [34603008, 35651584) append 0 append lockpos 0 00000080:00200000:2.0:1789762978.740721:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=34603008=0x2100000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762978.741042:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762978.741046:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 35651584 00000020:00010000:2.0:1789762978.741054:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762978.741057:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762978.741061:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762978.741065:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762978.741068:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762978.741076:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762978.741083:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762978.741107:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762978.741433:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 34603008<>35651584 00000100:00100000:2.0:1789762978.741471:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762978.741486:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577984:192.168.203.137@tcp:4:dd.500 00010000:00010000:2.0:1789762978.857912:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789762978.939328:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968184 gen 4 00000008:00010000:2.0:1789762978.939410:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762978.939418:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762978.939422:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762978.939424:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762978.939426:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762978.939428:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762978.939430:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762978.939432:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762978.939433:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762978.939435:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762978.939438:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762978.939440:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762978.939442:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762978.939444:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762978.939446:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762978.939447:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762978.939449:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762978.939451:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762978.939453:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762978.939455:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762978.939465:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762978.939467:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762978.939470:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762978.939472:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762978.939473:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762978.939475:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762978.939476:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762978.939477:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762978.939478:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762978.939479:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762978.939480:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762978.939481:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762978.939483:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762978.939484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762978.939485:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762978.939486:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762978.939487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762978.939488:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762978.939489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762978.939490:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762978.939491:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762978.939492:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762978.939493:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762978.939495:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762978.939496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762978.939498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762978.939499:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762978.939500:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762978.939501:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762978.939502:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762978.939503:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762978.939504:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762978.939505:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762978.939506:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762978.939507:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762978.939508:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762978.939509:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762978.939510:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762978.939511:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762978.939512:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762978.939513:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762978.939515:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762978.939516:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762978.939517:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762978.939518:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762978.939519:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762978.939520:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762978.939521:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762978.939522:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762978.939523:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762978.939524:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762978.939525:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762978.939526:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762978.939527:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762978.939528:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762978.939530:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762978.939531:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762978.939532:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762978.939534:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762978.939535:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762978.939536:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762978.939537:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762978.939538:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762978.939539:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762978.939540:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762978.939541:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762978.939542:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762978.939543:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762978.939545:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762978.939546:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762978.939547:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762978.939548:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762978.939549:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762978.939550:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762978.939551:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762978.939552:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762978.939553:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762978.939554:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762978.939555:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762978.939556:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762978.939557:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762978.939559:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762978.939560:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762978.939561:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762978.939562:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762978.939563:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762978.939564:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762978.939565:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762978.939567:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762978.939568:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762978.939570:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762978.939571:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762978.939572:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762978.939573:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762978.939574:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762978.939575:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762978.939576:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762978.939577:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762978.939578:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762978.939579:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762978.939580:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762978.939581:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762978.939582:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762978.939583:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762978.939584:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762978.939585:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762978.939586:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762978.939587:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762978.939588:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762978.939589:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762978.939590:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762978.939591:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762978.939593:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762978.939594:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762978.939595:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762978.939596:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762978.939597:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762978.939598:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762978.939599:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762978.939600:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762978.939602:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762978.939603:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762978.939604:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762978.939605:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762978.939606:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762978.939607:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762978.939608:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762978.939609:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762978.939610:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762978.939611:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762978.939613:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762978.939614:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762978.939615:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762978.939616:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762978.939617:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762978.939618:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762978.939619:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762978.939620:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762978.939621:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762978.939622:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762978.939623:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762978.939624:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762978.939625:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762978.939626:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762978.939627:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762978.939628:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762978.939629:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762978.939630:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762978.939631:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762978.939632:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762978.939634:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762978.939635:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762978.939636:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762978.939638:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762978.939639:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762978.939640:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762978.939641:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762978.939642:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762978.939643:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762978.939644:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762978.939645:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762978.939646:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762978.939647:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762978.939649:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762978.939650:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762978.939651:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762978.939652:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762978.939653:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762978.939654:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762978.939655:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762978.939656:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762978.939657:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762978.939658:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762978.939659:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762978.939660:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762978.939661:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762978.939662:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762978.939663:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762978.939664:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762978.939665:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762978.939666:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762978.939667:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762978.939669:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762978.939670:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762978.939672:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762978.939673:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762978.939674:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762978.939675:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762978.939676:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762978.939677:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762978.939678:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762978.939679:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762978.939680:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762978.939681:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762978.939682:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762978.939683:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762978.939684:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762978.939685:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762978.939686:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762978.939687:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762978.939688:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762978.939689:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762978.939690:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762978.939691:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762978.939692:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762978.939693:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762978.939694:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762978.939695:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762978.939696:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762978.939697:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762978.939698:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762978.939699:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762978.939700:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762978.939701:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762978.939703:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762978.939704:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762978.939705:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762978.939707:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762978.939709:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762978.939711:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762978.939713:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762978.939715:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762978.939716:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762978.939718:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762978.939720:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762978.939722:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762978.939724:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762978.939726:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762978.939728:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762978.939730:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762978.939732:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762978.939733:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762978.939735:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762978.939737:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762978.939739:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762978.939741:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762978.939743:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762978.939842:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762978.939857:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762978.939882:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067577984:192.168.203.137@tcp:4:dd.500 00000080:00200000:3.0:1789762978.939884:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [8448, 8703] 00000080:00200000:3.0:1789762978.939900:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1789762978.939905:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1789762978.939924:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 35651584 i_blocks 67591 00000020:00200000:3.0:1789762978.939927:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1789762978.939938:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1789762978.940178:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 35651584, count: 1048576 00000080:00200000:3.0:1789762978.940186:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 35651584, bytes: 1048576 00000080:00200000:3.0:1789762978.940195:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:3.0:1789762978.940204:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:3.0:1789762978.940207:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [35651584, 36700160) 0 0 00000080:00200000:3.0:1789762978.940212:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1789762978.940228:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [8704, 8959] 00020000:00200000:3.0:1789762978.940237:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 35651584 chunk: [35651584, 36700160) 36700160, 1048576 00020000:00200000:3.0:1789762978.940241:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1789762978.940280:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [8704, 8959] 00000008:00010000:3.0:1789762978.940308:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1789762978.940313:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9360061427b0, osc lock ffff936003e85300, flags 0x0 00000080:00200000:3.0:1789762978.940324:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [35651584, 36700160) append 0 append lockpos 0 00000080:00200000:3.0:1789762978.940335:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=35651584=0x2200000, pages 256 (max 131072), lockless, parallel 00000020:00200000:3.0:1789762978.940541:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1789762978.940544:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 36700160 00000020:00010000:3.0:1789762978.940552:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:3.0:1789762978.940556:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.940559:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061427b0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1789762978.940563:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:3.0:1789762978.940567:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061427b0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:3.0:1789762978.940575:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:3.0:1789762978.940583:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1789762978.940600:0:419:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff936010cdce40 00000008:00100000:3.0:1789762978.940865:0:419:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff936010aa4700 - object 0x280000400:15 offset 35651584<>36700160 00000100:00100000:3.0:1789762978.940894:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1789762978.940906:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578112:192.168.203.137@tcp:4:dd.500 00000100:00100000:3.0:1789762979.138882:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968185 gen 4 00000008:00010000:3.0:1789762979.138951:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdce40 00000020:00200000:3.0:1789762979.138958:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1789762979.138961:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1789762979.138963:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1789762979.138964:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1789762979.138965:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1789762979.138966:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1789762979.138967:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1789762979.138968:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1789762979.138969:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1789762979.138970:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1789762979.138972:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1789762979.138974:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1789762979.138975:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1789762979.138976:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1789762979.138978:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1789762979.138979:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1789762979.138980:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1789762979.138981:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1789762979.138982:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1789762979.138983:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1789762979.138984:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1789762979.138985:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1789762979.138986:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1789762979.138987:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1789762979.138988:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1789762979.138989:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1789762979.138990:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1789762979.138991:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1789762979.138992:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1789762979.138993:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1789762979.138994:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1789762979.138995:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1789762979.138996:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1789762979.138998:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1789762979.138999:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1789762979.139000:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1789762979.139001:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1789762979.139002:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1789762979.139003:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1789762979.139004:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1789762979.139005:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1789762979.139006:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1789762979.139008:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1789762979.139009:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1789762979.139011:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1789762979.139012:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1789762979.139014:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1789762979.139015:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1789762979.139016:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1789762979.139017:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1789762979.139018:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1789762979.139019:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1789762979.139020:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1789762979.139021:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1789762979.139022:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1789762979.139023:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1789762979.139024:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1789762979.139025:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1789762979.139026:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1789762979.139027:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1789762979.139028:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1789762979.139029:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1789762979.139030:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1789762979.139032:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1789762979.139033:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1789762979.139034:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1789762979.139035:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1789762979.139036:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1789762979.139037:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1789762979.139038:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1789762979.139039:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1789762979.139040:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1789762979.139041:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1789762979.139042:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1789762979.139043:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1789762979.139045:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1789762979.139046:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1789762979.139048:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1789762979.139049:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1789762979.139050:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1789762979.139051:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1789762979.139052:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1789762979.139053:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1789762979.139054:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1789762979.139056:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1789762979.139057:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1789762979.139057:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1789762979.139058:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1789762979.139059:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1789762979.139061:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1789762979.139062:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1789762979.139063:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1789762979.139064:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1789762979.139065:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1789762979.139066:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1789762979.139067:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1789762979.139068:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1789762979.139069:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1789762979.139070:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1789762979.139071:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1789762979.139072:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1789762979.139073:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1789762979.139074:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1789762979.139075:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1789762979.139076:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1789762979.139077:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1789762979.139079:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1789762979.139080:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1789762979.139082:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1789762979.139083:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1789762979.139085:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1789762979.139086:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1789762979.139087:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1789762979.139088:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1789762979.139089:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1789762979.139090:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1789762979.139091:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1789762979.139092:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1789762979.139093:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1789762979.139095:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1789762979.139096:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1789762979.139097:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1789762979.139098:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1789762979.139099:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1789762979.139100:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1789762979.139101:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1789762979.139102:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1789762979.139103:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1789762979.139104:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1789762979.139105:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1789762979.139106:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1789762979.139108:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1789762979.139109:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1789762979.139110:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1789762979.139111:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1789762979.139112:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1789762979.139113:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1789762979.139114:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1789762979.139116:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1789762979.139117:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1789762979.139119:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1789762979.139120:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1789762979.139121:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1789762979.139122:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1789762979.139123:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1789762979.139124:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1789762979.139125:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1789762979.139126:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1789762979.139127:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1789762979.139128:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1789762979.139129:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1789762979.139130:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1789762979.139132:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1789762979.139132:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1789762979.139133:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1789762979.139134:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1789762979.139135:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1789762979.139137:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1789762979.139138:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1789762979.139139:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1789762979.139140:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1789762979.139141:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1789762979.139142:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1789762979.139143:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1789762979.139145:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1789762979.139146:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1789762979.139147:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1789762979.139148:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1789762979.139149:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1789762979.139150:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1789762979.139152:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1789762979.139153:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1789762979.139155:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1789762979.139156:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1789762979.139157:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1789762979.139158:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1789762979.139159:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1789762979.139160:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1789762979.139161:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1789762979.139162:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1789762979.139163:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1789762979.139164:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1789762979.139165:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1789762979.139166:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1789762979.139167:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1789762979.139168:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1789762979.139169:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1789762979.139170:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1789762979.139171:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1789762979.139173:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1789762979.139173:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1789762979.139175:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1789762979.139176:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1789762979.139177:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1789762979.139178:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1789762979.139179:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1789762979.139180:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1789762979.139181:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1789762979.139182:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1789762979.139183:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1789762979.139184:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1789762979.139185:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1789762979.139187:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1789762979.139188:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1789762979.139190:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1789762979.139191:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1789762979.139192:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1789762979.139193:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1789762979.139194:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1789762979.139195:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1789762979.139196:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1789762979.139197:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1789762979.139198:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1789762979.139199:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1789762979.139200:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1789762979.139201:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1789762979.139202:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1789762979.139203:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1789762979.139205:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1789762979.139206:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1789762979.139207:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1789762979.139208:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1789762979.139209:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1789762979.139210:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1789762979.139211:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1789762979.139212:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1789762979.139213:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1789762979.139214:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1789762979.139215:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1789762979.139216:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1789762979.139217:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1789762979.139218:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1789762979.139219:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1789762979.139220:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1789762979.139222:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1789762979.139223:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1789762979.139224:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1789762979.139225:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1789762979.139226:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1789762979.139228:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1789762979.139229:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1789762979.139230:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1789762979.139231:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1789762979.139232:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1789762979.139233:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1789762979.139234:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1789762979.139235:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1789762979.139236:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1789762979.139237:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1789762979.139238:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1789762979.139239:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1789762979.139241:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1789762979.139242:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1789762979.139243:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1789762979.139244:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1789762979.139245:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1789762979.139332:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1789762979.139347:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdce40 00000100:00100000:3.0:1789762979.139371:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578112:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762979.139387:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [8704, 8959] 00000080:00200000:2.0:1789762979.139403:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762979.139408:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762979.139421:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 36700160 i_blocks 69639 00000020:00200000:2.0:1789762979.139424:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762979.139437:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762979.139644:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 36700160, count: 1048576 00000080:00200000:2.0:1789762979.139650:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 36700160, bytes: 1048576 00000080:00200000:2.0:1789762979.139657:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762979.139664:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762979.139667:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [36700160, 37748736) 0 0 00000080:00200000:2.0:1789762979.139670:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762979.139683:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [8960, 9215] 00020000:00200000:2.0:1789762979.139690:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 36700160 chunk: [36700160, 37748736) 37748736, 1048576 00020000:00200000:2.0:1789762979.139693:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762979.139709:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [8960, 9215] 00000008:00010000:2.0:1789762979.139722:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762979.139727:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762979.139737:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [36700160, 37748736) append 0 append lockpos 0 00000080:00200000:2.0:1789762979.139749:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=36700160=0x2300000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762979.139965:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762979.139968:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 37748736 00000020:00010000:2.0:1789762979.139974:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762979.139977:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762979.139980:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762979.139983:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762979.139985:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762979.139990:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762979.139995:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762979.140007:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762979.140238:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 36700160<>37748736 00000100:00100000:2.0:1789762979.140278:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762979.140290:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067578240:192.168.203.137@tcp:4:dd.500 00000100:00100000:2.0:1789762979.339073:0:2397:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968186 gen 4 00000008:00010000:2.0:1789762979.339144:0:2397:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff93601a625840 00000020:00200000:2.0:1789762979.339152:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:2.0:1789762979.339155:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:2.0:1789762979.339156:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:2.0:1789762979.339158:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:2.0:1789762979.339159:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:2.0:1789762979.339160:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:2.0:1789762979.339161:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:2.0:1789762979.339162:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:2.0:1789762979.339163:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:2.0:1789762979.339165:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:2.0:1789762979.339166:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:2.0:1789762979.339167:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:2.0:1789762979.339168:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:2.0:1789762979.339169:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:2.0:1789762979.339170:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:2.0:1789762979.339171:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:2.0:1789762979.339172:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:2.0:1789762979.339183:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:2.0:1789762979.339184:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:2.0:1789762979.339186:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:2.0:1789762979.339187:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:2.0:1789762979.339188:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:2.0:1789762979.339189:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:2.0:1789762979.339190:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:2.0:1789762979.339191:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:2.0:1789762979.339192:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:2.0:1789762979.339193:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:2.0:1789762979.339194:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:2.0:1789762979.339195:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:2.0:1789762979.339196:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:2.0:1789762979.339197:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:2.0:1789762979.339198:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:2.0:1789762979.339200:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:2.0:1789762979.339201:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:2.0:1789762979.339202:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:2.0:1789762979.339204:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:2.0:1789762979.339205:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:2.0:1789762979.339206:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:2.0:1789762979.339207:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:2.0:1789762979.339209:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:2.0:1789762979.339210:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:2.0:1789762979.339211:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:2.0:1789762979.339212:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:2.0:1789762979.339213:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:2.0:1789762979.339214:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:2.0:1789762979.339215:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:2.0:1789762979.339216:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:2.0:1789762979.339217:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:2.0:1789762979.339218:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:2.0:1789762979.339219:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:2.0:1789762979.339220:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:2.0:1789762979.339221:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:2.0:1789762979.339222:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:2.0:1789762979.339223:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:2.0:1789762979.339224:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:2.0:1789762979.339225:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:2.0:1789762979.339226:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:2.0:1789762979.339227:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:2.0:1789762979.339228:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:2.0:1789762979.339229:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:2.0:1789762979.339230:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:2.0:1789762979.339231:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:2.0:1789762979.339232:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:2.0:1789762979.339233:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:2.0:1789762979.339235:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:2.0:1789762979.339236:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:2.0:1789762979.339238:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:2.0:1789762979.339239:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:2.0:1789762979.339240:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:2.0:1789762979.339242:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:2.0:1789762979.339243:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:2.0:1789762979.339244:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:2.0:1789762979.339245:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:2.0:1789762979.339246:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:2.0:1789762979.339247:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:2.0:1789762979.339248:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:2.0:1789762979.339249:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:2.0:1789762979.339251:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:2.0:1789762979.339266:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:2.0:1789762979.339268:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:2.0:1789762979.339269:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:2.0:1789762979.339270:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:2.0:1789762979.339271:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:2.0:1789762979.339272:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:2.0:1789762979.339273:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:2.0:1789762979.339274:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:2.0:1789762979.339275:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:2.0:1789762979.339276:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:2.0:1789762979.339288:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:2.0:1789762979.339289:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:2.0:1789762979.339290:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:2.0:1789762979.339291:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:2.0:1789762979.339292:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:2.0:1789762979.339293:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:2.0:1789762979.339294:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:2.0:1789762979.339295:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:2.0:1789762979.339297:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:2.0:1789762979.339298:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:2.0:1789762979.339299:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:2.0:1789762979.339301:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:2.0:1789762979.339302:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:2.0:1789762979.339303:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:2.0:1789762979.339304:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:2.0:1789762979.339305:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:2.0:1789762979.339306:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:2.0:1789762979.339307:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:2.0:1789762979.339308:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:2.0:1789762979.339309:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:2.0:1789762979.339310:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:2.0:1789762979.339311:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:2.0:1789762979.339313:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:2.0:1789762979.339314:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:2.0:1789762979.339315:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:2.0:1789762979.339316:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:2.0:1789762979.339317:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:2.0:1789762979.339319:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:2.0:1789762979.339320:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:2.0:1789762979.339322:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:2.0:1789762979.339324:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:2.0:1789762979.339325:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:2.0:1789762979.339327:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:2.0:1789762979.339328:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:2.0:1789762979.339330:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:2.0:1789762979.339331:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:2.0:1789762979.339333:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:2.0:1789762979.339334:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:2.0:1789762979.339336:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:2.0:1789762979.339338:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:2.0:1789762979.339340:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:2.0:1789762979.339342:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:2.0:1789762979.339344:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:2.0:1789762979.339346:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:2.0:1789762979.339348:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:2.0:1789762979.339350:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:2.0:1789762979.339352:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:2.0:1789762979.339354:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:2.0:1789762979.339356:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:2.0:1789762979.339358:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:2.0:1789762979.339360:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:2.0:1789762979.339362:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:2.0:1789762979.339364:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:2.0:1789762979.339366:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:2.0:1789762979.339368:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:2.0:1789762979.339370:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:2.0:1789762979.339372:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:2.0:1789762979.339375:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:2.0:1789762979.339377:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:2.0:1789762979.339379:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:2.0:1789762979.339381:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:2.0:1789762979.339383:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:2.0:1789762979.339385:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:2.0:1789762979.339387:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:2.0:1789762979.339389:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:2.0:1789762979.339391:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:2.0:1789762979.339393:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:2.0:1789762979.339395:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:2.0:1789762979.339397:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:2.0:1789762979.339400:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:2.0:1789762979.339402:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:2.0:1789762979.339404:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:2.0:1789762979.339406:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:2.0:1789762979.339408:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:2.0:1789762979.339410:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:2.0:1789762979.339412:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:2.0:1789762979.339413:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:2.0:1789762979.339414:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:2.0:1789762979.339415:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:2.0:1789762979.339416:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:2.0:1789762979.339417:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:2.0:1789762979.339418:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:2.0:1789762979.339419:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:2.0:1789762979.339420:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:2.0:1789762979.339422:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:2.0:1789762979.339422:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:2.0:1789762979.339423:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:2.0:1789762979.339424:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:2.0:1789762979.339425:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:2.0:1789762979.339426:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:2.0:1789762979.339427:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:2.0:1789762979.339428:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:2.0:1789762979.339430:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:2.0:1789762979.339431:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:2.0:1789762979.339432:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:2.0:1789762979.339433:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:2.0:1789762979.339434:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:2.0:1789762979.339435:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:2.0:1789762979.339436:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:2.0:1789762979.339437:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:2.0:1789762979.339438:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:2.0:1789762979.339439:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:2.0:1789762979.339440:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:2.0:1789762979.339441:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:2.0:1789762979.339443:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:2.0:1789762979.339444:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:2.0:1789762979.339446:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:2.0:1789762979.339447:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:2.0:1789762979.339448:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:2.0:1789762979.339449:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:2.0:1789762979.339450:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:2.0:1789762979.339451:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:2.0:1789762979.339452:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:2.0:1789762979.339454:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:2.0:1789762979.339455:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:2.0:1789762979.339456:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:2.0:1789762979.339457:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:2.0:1789762979.339458:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:2.0:1789762979.339459:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:2.0:1789762979.339460:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:2.0:1789762979.339461:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:2.0:1789762979.339462:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:2.0:1789762979.339463:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:2.0:1789762979.339464:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:2.0:1789762979.339465:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:2.0:1789762979.339466:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:2.0:1789762979.339467:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:2.0:1789762979.339468:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:2.0:1789762979.339469:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:2.0:1789762979.339470:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:2.0:1789762979.339471:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:2.0:1789762979.339472:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:2.0:1789762979.339473:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:2.0:1789762979.339474:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:2.0:1789762979.339475:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:2.0:1789762979.339476:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:2.0:1789762979.339478:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:2.0:1789762979.339479:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:2.0:1789762979.339481:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:2.0:1789762979.339482:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:2.0:1789762979.339483:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:2.0:1789762979.339484:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:2.0:1789762979.339485:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:2.0:1789762979.339486:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:2.0:1789762979.339487:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:2.0:1789762979.339488:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:2.0:1789762979.339489:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:2.0:1789762979.339490:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:2.0:1789762979.339491:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:2.0:1789762979.339492:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:2.0:1789762979.339494:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:2.0:1789762979.339495:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:2.0:1789762979.339496:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:2.0:1789762979.339497:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:2.0:1789762979.339498:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:2.0:1789762979.339499:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:2.0:1789762979.339500:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:2.0:1789762979.339501:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:2.0:1789762979.339502:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:2.0:1789762979.339503:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:2.0:1789762979.339504:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:2.0:1789762979.339505:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:2.0:1789762979.339506:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:2.0:1789762979.339507:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:2.0:1789762979.339508:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:2.0:1789762979.339509:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:2.0:1789762979.339511:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:2.0:1789762979.339512:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789762979.339578:0:2397:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789762979.339595:0:2397:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff93601a625840 00000100:00100000:2.0:1789762979.339627:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067578240:192.168.203.137@tcp:4:dd.500 00000080:00200000:3.0:1789762979.339654:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [8960, 9215] 00000080:00200000:3.0:1789762979.339670:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1789762979.339675:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:3.0:1789762979.339688:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 37748736 i_blocks 71687 00000020:00200000:3.0:1789762979.339691:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:3.0:1789762979.339699:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:3.0:1789762979.339918:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 37748736, count: 1048576 00000080:00200000:3.0:1789762979.339924:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 37748736, bytes: 1048576 00000080:00200000:3.0:1789762979.339932:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:3.0:1789762979.339940:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:3.0:1789762979.339942:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [37748736, 38797312) 0 0 00000080:00200000:3.0:1789762979.339946:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1789762979.339960:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [9216, 9471] 00020000:00200000:3.0:1789762979.339967:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 37748736 chunk: [37748736, 38797312) 38797312, 1048576 00020000:00200000:3.0:1789762979.339970:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1789762979.339998:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [9216, 9471] 00000008:00010000:3.0:1789762979.340011:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:3.0:1789762979.340016:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff9360061427b0, osc lock ffff936003e85300, flags 0x0 00000080:00200000:3.0:1789762979.340026:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [37748736, 38797312) append 0 append lockpos 0 00000080:00200000:3.0:1789762979.340037:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=37748736=0x2400000, pages 256 (max 131072), lockless, parallel 00000020:00200000:3.0:1789762979.340249:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:3.0:1789762979.340251:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 38797312 00000020:00010000:3.0:1789762979.340273:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:3.0:1789762979.340276:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:3.0:1789762979.340289:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff9360061427b0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:3.0:1789762979.340291:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:3.0:1789762979.340294:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff9360061427b0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:3.0:1789762979.340299:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:3.0:1789762979.340304:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:3.0:1789762979.340318:0:419:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff936010cdce40 00000008:00100000:3.0:1789762979.340553:0:419:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff936010aa4700 - object 0x280000400:15 offset 37748736<>38797312 00000100:00100000:3.0:1789762979.340595:0:2398:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:3.0:1789762979.340608:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578368:192.168.203.137@tcp:4:dd.500 00000100:00100000:3.0:1789762979.539512:0:2398:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff936020592000: committing for last_committed 4294968187 gen 4 00000008:00010000:3.0:1789762979.539567:0:2398:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff936010cdce40 00000020:00200000:3.0:1789762979.539574:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 255 items remaining. 00000020:00200000:3.0:1789762979.539577:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 254 items remaining. 00000020:00200000:3.0:1789762979.539579:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 253 items remaining. 00000020:00200000:3.0:1789762979.539580:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 252 items remaining. 00000020:00200000:3.0:1789762979.539582:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 251 items remaining. 00000020:00200000:3.0:1789762979.539583:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 250 items remaining. 00000020:00200000:3.0:1789762979.539584:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 249 items remaining. 00000020:00200000:3.0:1789762979.539585:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 248 items remaining. 00000020:00200000:3.0:1789762979.539586:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 247 items remaining. 00000020:00200000:3.0:1789762979.539587:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 246 items remaining. 00000020:00200000:3.0:1789762979.539588:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 245 items remaining. 00000020:00200000:3.0:1789762979.539590:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 244 items remaining. 00000020:00200000:3.0:1789762979.539591:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 243 items remaining. 00000020:00200000:3.0:1789762979.539592:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 242 items remaining. 00000020:00200000:3.0:1789762979.539593:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 241 items remaining. 00000020:00200000:3.0:1789762979.539594:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 240 items remaining. 00000020:00200000:3.0:1789762979.539595:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 239 items remaining. 00000020:00200000:3.0:1789762979.539596:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 238 items remaining. 00000020:00200000:3.0:1789762979.539597:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 237 items remaining. 00000020:00200000:3.0:1789762979.539598:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 236 items remaining. 00000020:00200000:3.0:1789762979.539599:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 235 items remaining. 00000020:00200000:3.0:1789762979.539600:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 234 items remaining. 00000020:00200000:3.0:1789762979.539601:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 233 items remaining. 00000020:00200000:3.0:1789762979.539602:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 232 items remaining. 00000020:00200000:3.0:1789762979.539603:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 231 items remaining. 00000020:00200000:3.0:1789762979.539604:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 230 items remaining. 00000020:00200000:3.0:1789762979.539605:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 229 items remaining. 00000020:00200000:3.0:1789762979.539606:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 228 items remaining. 00000020:00200000:3.0:1789762979.539607:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 227 items remaining. 00000020:00200000:3.0:1789762979.539608:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 226 items remaining. 00000020:00200000:3.0:1789762979.539609:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 225 items remaining. 00000020:00200000:3.0:1789762979.539610:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 224 items remaining. 00000020:00200000:3.0:1789762979.539611:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 223 items remaining. 00000020:00200000:3.0:1789762979.539613:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 222 items remaining. 00000020:00200000:3.0:1789762979.539615:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 221 items remaining. 00000020:00200000:3.0:1789762979.539616:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 220 items remaining. 00000020:00200000:3.0:1789762979.539617:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 219 items remaining. 00000020:00200000:3.0:1789762979.539618:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 218 items remaining. 00000020:00200000:3.0:1789762979.539620:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 217 items remaining. 00000020:00200000:3.0:1789762979.539621:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 216 items remaining. 00000020:00200000:3.0:1789762979.539622:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 215 items remaining. 00000020:00200000:3.0:1789762979.539623:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 214 items remaining. 00000020:00200000:3.0:1789762979.539624:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 213 items remaining. 00000020:00200000:3.0:1789762979.539625:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 212 items remaining. 00000020:00200000:3.0:1789762979.539626:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 211 items remaining. 00000020:00200000:3.0:1789762979.539627:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 210 items remaining. 00000020:00200000:3.0:1789762979.539628:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 209 items remaining. 00000020:00200000:3.0:1789762979.539629:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 208 items remaining. 00000020:00200000:3.0:1789762979.539630:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 207 items remaining. 00000020:00200000:3.0:1789762979.539631:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 206 items remaining. 00000020:00200000:3.0:1789762979.539633:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 205 items remaining. 00000020:00200000:3.0:1789762979.539634:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 204 items remaining. 00000020:00200000:3.0:1789762979.539635:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 203 items remaining. 00000020:00200000:3.0:1789762979.539636:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 202 items remaining. 00000020:00200000:3.0:1789762979.539637:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 201 items remaining. 00000020:00200000:3.0:1789762979.539638:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 200 items remaining. 00000020:00200000:3.0:1789762979.539639:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 199 items remaining. 00000020:00200000:3.0:1789762979.539640:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 198 items remaining. 00000020:00200000:3.0:1789762979.539641:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 197 items remaining. 00000020:00200000:3.0:1789762979.539642:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 196 items remaining. 00000020:00200000:3.0:1789762979.539643:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 195 items remaining. 00000020:00200000:3.0:1789762979.539644:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 194 items remaining. 00000020:00200000:3.0:1789762979.539645:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 193 items remaining. 00000020:00200000:3.0:1789762979.539646:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 192 items remaining. 00000020:00200000:3.0:1789762979.539647:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 191 items remaining. 00000020:00200000:3.0:1789762979.539649:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 190 items remaining. 00000020:00200000:3.0:1789762979.539650:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 189 items remaining. 00000020:00200000:3.0:1789762979.539652:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 188 items remaining. 00000020:00200000:3.0:1789762979.539653:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 187 items remaining. 00000020:00200000:3.0:1789762979.539654:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 186 items remaining. 00000020:00200000:3.0:1789762979.539655:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 185 items remaining. 00000020:00200000:3.0:1789762979.539656:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 184 items remaining. 00000020:00200000:3.0:1789762979.539657:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 183 items remaining. 00000020:00200000:3.0:1789762979.539658:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 182 items remaining. 00000020:00200000:3.0:1789762979.539659:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 181 items remaining. 00000020:00200000:3.0:1789762979.539660:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 180 items remaining. 00000020:00200000:3.0:1789762979.539662:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 179 items remaining. 00000020:00200000:3.0:1789762979.539663:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 178 items remaining. 00000020:00200000:3.0:1789762979.539664:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 177 items remaining. 00000020:00200000:3.0:1789762979.539665:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 176 items remaining. 00000020:00200000:3.0:1789762979.539666:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 175 items remaining. 00000020:00200000:3.0:1789762979.539667:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 174 items remaining. 00000020:00200000:3.0:1789762979.539668:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 173 items remaining. 00000020:00200000:3.0:1789762979.539669:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 172 items remaining. 00000020:00200000:3.0:1789762979.539670:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 171 items remaining. 00000020:00200000:3.0:1789762979.539671:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 170 items remaining. 00000020:00200000:3.0:1789762979.539672:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 169 items remaining. 00000020:00200000:3.0:1789762979.539673:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 168 items remaining. 00000020:00200000:3.0:1789762979.539674:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 167 items remaining. 00000020:00200000:3.0:1789762979.539675:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 166 items remaining. 00000020:00200000:3.0:1789762979.539676:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 165 items remaining. 00000020:00200000:3.0:1789762979.539677:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 164 items remaining. 00000020:00200000:3.0:1789762979.539678:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 163 items remaining. 00000020:00200000:3.0:1789762979.539679:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 162 items remaining. 00000020:00200000:3.0:1789762979.539688:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 161 items remaining. 00000020:00200000:3.0:1789762979.539689:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 160 items remaining. 00000020:00200000:3.0:1789762979.539690:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 159 items remaining. 00000020:00200000:3.0:1789762979.539692:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 158 items remaining. 00000020:00200000:3.0:1789762979.539694:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 157 items remaining. 00000020:00200000:3.0:1789762979.539695:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 156 items remaining. 00000020:00200000:3.0:1789762979.539697:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 155 items remaining. 00000020:00200000:3.0:1789762979.539698:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 154 items remaining. 00000020:00200000:3.0:1789762979.539699:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 153 items remaining. 00000020:00200000:3.0:1789762979.539700:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 152 items remaining. 00000020:00200000:3.0:1789762979.539701:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 151 items remaining. 00000020:00200000:3.0:1789762979.539702:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 150 items remaining. 00000020:00200000:3.0:1789762979.539703:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 149 items remaining. 00000020:00200000:3.0:1789762979.539704:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 148 items remaining. 00000020:00200000:3.0:1789762979.539705:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 147 items remaining. 00000020:00200000:3.0:1789762979.539706:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 146 items remaining. 00000020:00200000:3.0:1789762979.539707:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 145 items remaining. 00000020:00200000:3.0:1789762979.539708:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 144 items remaining. 00000020:00200000:3.0:1789762979.539709:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 143 items remaining. 00000020:00200000:3.0:1789762979.539710:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 142 items remaining. 00000020:00200000:3.0:1789762979.539711:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 141 items remaining. 00000020:00200000:3.0:1789762979.539712:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 140 items remaining. 00000020:00200000:3.0:1789762979.539714:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 139 items remaining. 00000020:00200000:3.0:1789762979.539715:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 138 items remaining. 00000020:00200000:3.0:1789762979.539716:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 137 items remaining. 00000020:00200000:3.0:1789762979.539717:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 136 items remaining. 00000020:00200000:3.0:1789762979.539718:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 135 items remaining. 00000020:00200000:3.0:1789762979.539719:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 134 items remaining. 00000020:00200000:3.0:1789762979.539720:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 133 items remaining. 00000020:00200000:3.0:1789762979.539721:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 132 items remaining. 00000020:00200000:3.0:1789762979.539722:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 131 items remaining. 00000020:00200000:3.0:1789762979.539723:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 130 items remaining. 00000020:00200000:3.0:1789762979.539724:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 129 items remaining. 00000020:00200000:3.0:1789762979.539725:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 128 items remaining. 00000020:00200000:3.0:1789762979.539726:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 127 items remaining. 00000020:00200000:3.0:1789762979.539728:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 126 items remaining. 00000020:00200000:3.0:1789762979.539730:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 125 items remaining. 00000020:00200000:3.0:1789762979.539731:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 124 items remaining. 00000020:00200000:3.0:1789762979.539732:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 123 items remaining. 00000020:00200000:3.0:1789762979.539734:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 122 items remaining. 00000020:00200000:3.0:1789762979.539735:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 121 items remaining. 00000020:00200000:3.0:1789762979.539736:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 120 items remaining. 00000020:00200000:3.0:1789762979.539737:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 119 items remaining. 00000020:00200000:3.0:1789762979.539738:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 118 items remaining. 00000020:00200000:3.0:1789762979.539739:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 117 items remaining. 00000020:00200000:3.0:1789762979.539740:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 116 items remaining. 00000020:00200000:3.0:1789762979.539741:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 115 items remaining. 00000020:00200000:3.0:1789762979.539742:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 114 items remaining. 00000020:00200000:3.0:1789762979.539743:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 113 items remaining. 00000020:00200000:3.0:1789762979.539744:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 112 items remaining. 00000020:00200000:3.0:1789762979.539746:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 111 items remaining. 00000020:00200000:3.0:1789762979.539747:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 110 items remaining. 00000020:00200000:3.0:1789762979.539748:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 109 items remaining. 00000020:00200000:3.0:1789762979.539749:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 108 items remaining. 00000020:00200000:3.0:1789762979.539750:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 107 items remaining. 00000020:00200000:3.0:1789762979.539751:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 106 items remaining. 00000020:00200000:3.0:1789762979.539752:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 105 items remaining. 00000020:00200000:3.0:1789762979.539753:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 104 items remaining. 00000020:00200000:3.0:1789762979.539754:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 103 items remaining. 00000020:00200000:3.0:1789762979.539755:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 102 items remaining. 00000020:00200000:3.0:1789762979.539756:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 101 items remaining. 00000020:00200000:3.0:1789762979.539757:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 100 items remaining. 00000020:00200000:3.0:1789762979.539758:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 99 items remaining. 00000020:00200000:3.0:1789762979.539759:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 98 items remaining. 00000020:00200000:3.0:1789762979.539760:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 97 items remaining. 00000020:00200000:3.0:1789762979.539761:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 96 items remaining. 00000020:00200000:3.0:1789762979.539762:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 95 items remaining. 00000020:00200000:3.0:1789762979.539764:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 94 items remaining. 00000020:00200000:3.0:1789762979.539765:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 93 items remaining. 00000020:00200000:3.0:1789762979.539767:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 92 items remaining. 00000020:00200000:3.0:1789762979.539768:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 91 items remaining. 00000020:00200000:3.0:1789762979.539769:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 90 items remaining. 00000020:00200000:3.0:1789762979.539770:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 89 items remaining. 00000020:00200000:3.0:1789762979.539771:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 88 items remaining. 00000020:00200000:3.0:1789762979.539772:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 87 items remaining. 00000020:00200000:3.0:1789762979.539774:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 86 items remaining. 00000020:00200000:3.0:1789762979.539775:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 85 items remaining. 00000020:00200000:3.0:1789762979.539776:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 84 items remaining. 00000020:00200000:3.0:1789762979.539777:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 83 items remaining. 00000020:00200000:3.0:1789762979.539778:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 82 items remaining. 00000020:00200000:3.0:1789762979.539779:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 81 items remaining. 00000020:00200000:3.0:1789762979.539780:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 80 items remaining. 00000020:00200000:3.0:1789762979.539781:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 79 items remaining. 00000020:00200000:3.0:1789762979.539782:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 78 items remaining. 00000020:00200000:3.0:1789762979.539783:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 77 items remaining. 00000020:00200000:3.0:1789762979.539784:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 76 items remaining. 00000020:00200000:3.0:1789762979.539785:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 75 items remaining. 00000020:00200000:3.0:1789762979.539786:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 74 items remaining. 00000020:00200000:3.0:1789762979.539787:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 73 items remaining. 00000020:00200000:3.0:1789762979.539788:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 72 items remaining. 00000020:00200000:3.0:1789762979.539789:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 71 items remaining. 00000020:00200000:3.0:1789762979.539790:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 70 items remaining. 00000020:00200000:3.0:1789762979.539791:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 69 items remaining. 00000020:00200000:3.0:1789762979.539792:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 68 items remaining. 00000020:00200000:3.0:1789762979.539793:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 67 items remaining. 00000020:00200000:3.0:1789762979.539794:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 66 items remaining. 00000020:00200000:3.0:1789762979.539813:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 65 items remaining. 00000020:00200000:3.0:1789762979.539815:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 64 items remaining. 00000020:00200000:3.0:1789762979.539816:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 63 items remaining. 00000020:00200000:3.0:1789762979.539818:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 62 items remaining. 00000020:00200000:3.0:1789762979.539819:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 61 items remaining. 00000020:00200000:3.0:1789762979.539821:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 60 items remaining. 00000020:00200000:3.0:1789762979.539822:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 59 items remaining. 00000020:00200000:3.0:1789762979.539823:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 58 items remaining. 00000020:00200000:3.0:1789762979.539825:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 57 items remaining. 00000020:00200000:3.0:1789762979.539826:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 56 items remaining. 00000020:00200000:3.0:1789762979.539827:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 55 items remaining. 00000020:00200000:3.0:1789762979.539828:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 54 items remaining. 00000020:00200000:3.0:1789762979.539829:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 53 items remaining. 00000020:00200000:3.0:1789762979.539830:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 52 items remaining. 00000020:00200000:3.0:1789762979.539831:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 51 items remaining. 00000020:00200000:3.0:1789762979.539832:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 50 items remaining. 00000020:00200000:3.0:1789762979.539833:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 49 items remaining. 00000020:00200000:3.0:1789762979.539834:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 48 items remaining. 00000020:00200000:3.0:1789762979.539835:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 47 items remaining. 00000020:00200000:3.0:1789762979.539836:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 46 items remaining. 00000020:00200000:3.0:1789762979.539837:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 45 items remaining. 00000020:00200000:3.0:1789762979.539838:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 44 items remaining. 00000020:00200000:3.0:1789762979.539839:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 43 items remaining. 00000020:00200000:3.0:1789762979.539840:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 42 items remaining. 00000020:00200000:3.0:1789762979.539841:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 41 items remaining. 00000020:00200000:3.0:1789762979.539842:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 40 items remaining. 00000020:00200000:3.0:1789762979.539843:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 39 items remaining. 00000020:00200000:3.0:1789762979.539844:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 38 items remaining. 00000020:00200000:3.0:1789762979.539845:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 37 items remaining. 00000020:00200000:3.0:1789762979.539846:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 36 items remaining. 00000020:00200000:3.0:1789762979.539847:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 35 items remaining. 00000020:00200000:3.0:1789762979.539848:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 34 items remaining. 00000020:00200000:3.0:1789762979.539849:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 33 items remaining. 00000020:00200000:3.0:1789762979.539850:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 32 items remaining. 00000020:00200000:3.0:1789762979.539851:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 31 items remaining. 00000020:00200000:3.0:1789762979.539853:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 30 items remaining. 00000020:00200000:3.0:1789762979.539854:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 29 items remaining. 00000020:00200000:3.0:1789762979.539855:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 28 items remaining. 00000020:00200000:3.0:1789762979.539857:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 27 items remaining. 00000020:00200000:3.0:1789762979.539858:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 26 items remaining. 00000020:00200000:3.0:1789762979.539859:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 25 items remaining. 00000020:00200000:3.0:1789762979.539860:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 24 items remaining. 00000020:00200000:3.0:1789762979.539861:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 23 items remaining. 00000020:00200000:3.0:1789762979.539862:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 22 items remaining. 00000020:00200000:3.0:1789762979.539863:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 21 items remaining. 00000020:00200000:3.0:1789762979.539864:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 20 items remaining. 00000020:00200000:3.0:1789762979.539865:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 19 items remaining. 00000020:00200000:3.0:1789762979.539867:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 18 items remaining. 00000020:00200000:3.0:1789762979.539868:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 17 items remaining. 00000020:00200000:3.0:1789762979.539869:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 16 items remaining. 00000020:00200000:3.0:1789762979.539870:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 15 items remaining. 00000020:00200000:3.0:1789762979.539871:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 14 items remaining. 00000020:00200000:3.0:1789762979.539872:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 13 items remaining. 00000020:00200000:3.0:1789762979.539873:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 12 items remaining. 00000020:00200000:3.0:1789762979.539874:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 11 items remaining. 00000020:00200000:3.0:1789762979.539875:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 10 items remaining. 00000020:00200000:3.0:1789762979.539876:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 9 items remaining. 00000020:00200000:3.0:1789762979.539877:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 8 items remaining. 00000020:00200000:3.0:1789762979.539877:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 7 items remaining. 00000020:00200000:3.0:1789762979.539878:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 6 items remaining. 00000020:00200000:3.0:1789762979.539879:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 5 items remaining. 00000020:00200000:3.0:1789762979.539880:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 4 items remaining. 00000020:00200000:3.0:1789762979.539882:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 3 items remaining. 00000020:00200000:3.0:1789762979.539883:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 2 items remaining. 00000020:00200000:3.0:1789762979.539884:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000020:00200000:3.0:1789762979.539885:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:3.0:1789762979.539950:0:2398:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:3.0:1789762979.539966:0:2398:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff936010cdce40 00000100:00100000:3.0:1789762979.539990:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578368:192.168.203.137@tcp:4:dd.500 00000080:00200000:2.0:1789762979.539998:0:16340:0:(file.c:2205:ll_file_io_generic()) Range unlock [9216, 9471] 00000080:00200000:2.0:1789762979.540014:0:16340:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789762979.540020:0:16340:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 1048576, restart: 0 00000080:00200000:2.0:1789762979.540032:0:16340:0:(file.c:1775:ll_merge_attr_nolock()) [0x240000402:0x1a:0x0] updating i_size 38797312 i_blocks 73735 00000020:00200000:2.0:1789762979.540036:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789762979.540047:0:16340:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 1048576 00000080:00200008:2.0:1789762979.540267:0:16340:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x240000402:0x1a:0x0], ppos: 38797312, count: 1048576 00000080:00200000:2.0:1789762979.540274:0:16340:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 38797312, bytes: 1048576 00000080:00200000:2.0:1789762979.540283:0:16340:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789762979.540291:0:16340:0:(cl_io.c:214:cl_io_rw_init()) header@000000005296828e[0x110, 2, [0x240000402:0x1a:0x0] hash] 00000020:00200000:2.0:1789762979.540293:0:16340:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [38797312, 39845888) 0 0 00000080:00200000:2.0:1789762979.540297:0:16340:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789762979.540311:0:16340:0:(file.c:2168:ll_file_io_generic()) Range lock [9472, 9727] 00020000:00200000:2.0:1789762979.540317:0:16340:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 38797312 chunk: [38797312, 39845888) 39845888, 1048576 00020000:00200000:2.0:1789762979.540321:0:16340:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789762979.540337:0:16340:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [9472, 9727] 00000008:00010000:2.0:1789762979.540349:0:16340:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789762979.540354:0:16340:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff936008870db0, osc lock ffff9360183de5f0, flags 0x0 00000080:00200000:2.0:1789762979.540365:0:16340:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [38797312, 39845888) append 0 append lockpos 0 00000080:00200000:2.0:1789762979.540378:0:16340:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x240000402:0x1a:0x0](ffff936018d980c8), size=1048576 (max 536870912), offset=38797312=0x2500000, pages 256 (max 131072), lockless, parallel 00000020:00200000:2.0:1789762979.540575:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 256 items remaining. 00000080:00200000:2.0:1789762979.540578:0:16340:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 1048576 count 1048576 file_offset 39845888 00000020:00010000:2.0:1789762979.540585:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_release():245 00000020:00010000:2.0:1789762979.540588:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789762979.540591:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff936008870db0 (ffff936008beacf8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789762979.540594:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936010e3a218 (ffff936008bea758/0) at cl_lock_fini():72 00000020:00010000:2.0:1789762979.540596:0:16340:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff936008870db0 (ffff936008bea758/1) at cl_lock_fini():72 00000080:00200000:2.0:1789762979.540602:0:16340:0:(vvp_io.c:533:vvp_io_advance()) advancing 1048576 bytes 00000020:00200000:2.0:1789762979.540607:0:16340:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789762979.540620:0:255:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff93601a625840 00000008:00100000:2.0:1789762979.540884:0:255:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff9360061dea00 - object 0x280000400:15 offset 38797312<>39845888 00000100:00100000:2.0:1789762979.540917:0:2397:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789762979.540929:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067578496:192.168.203.137@tcp:4:dd.500 00010000:00010000:3.0:1789762979.881965:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762980.905981:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789762981.162226:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0607efc5-93ac-4424-8fa4-2e162e139892:2397:1876702067578624:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762981.162233:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578752:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762981.162456:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067579008:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762981.162460:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067578880:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762981.162550:0:2398:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2398:1876702067579264:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762981.162558:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067579136:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762981.162640:0:2397:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2397:1876702067579392:192.168.203.137@tcp:400:kworker.0 00010000:00010000:2.0:1789762981.929990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762982.954028:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762983.977970:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762985.001940:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762986.026041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762987.049902:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762988.073900:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762989.098062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762990.121888:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762991.145932:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762992.170083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762993.193954:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762994.217888:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762995.242019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1789762995.497944:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762979/real 1789762979] req@ffff9360061dea00 x1876702067578496/t0(0) o4->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:6/4 lens 488/448 e 0 to 1 dl 1789762995 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'dd.500' uid:500 gid:500 projid:0 00000100:00080000:3.0:1789762995.510029:0:2397:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789762995.510036:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff936020592000: Connection to lustre-OST0000 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789762995.516514:0:2397:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0000_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1789762995.516778:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067581696:192.168.203.137@tcp:8:kworker.0 00010000:00010000:2.0:1789762996.265903:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762997.289907:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1789762997.545929:0:2398:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061dca80 x1876702067578752/t0(0) o400->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000400:3.0:1789762997.545929:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061dfb80 x1876702067578624/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.545982:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061dca80 x1876702067578752/t0(0) o400->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.546003:0:2398:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1789762997.546007:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff936020592000: Connection to lustre-MDT0000 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00100000:3.0:1789762997.552260:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061dfb80 x1876702067578624/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762997.552273:0:2397:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1789762997.552277:0:2397:0:(import.c:186:ptlrpc_set_import_discon()) MGC192.168.203.137@tcp: Connection to MGS (at 192.168.203.137@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1789762997.555578:0:2398:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:2.0:1789762997.555595:0:2398:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061df800 x1876702067579008/t0(0) o400->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.555611:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061df800 x1876702067579008/t0(0) o400->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.555618:0:2398:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff93601a4b7000 already not connected (conn 5, was 4): CONNECTING 00000100:00000400:2.0:1789762997.555623:0:2398:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061dc000 x1876702067579264/t0(0) o400->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.555631:0:2398:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061dc000 x1876702067579264/t0(0) o400->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762997.555637:0:2398:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:2.0:1789762997.555639:0:2398:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff936008b5a800: Connection to lustre-MDT0000 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:2.0:1789762997.555641:0:2398:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00100000:2.0:1789762997.555655:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067578752:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762997.555659:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2398:1876702067579008:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762997.555663:0:2398:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2398:1876702067579264:192.168.203.137@tcp:400:kworker.0 00000100:00080000:3.0:1789762997.560214:0:2397:0:(import.c:192:ptlrpc_set_import_discon()) import MGS@192.168.203.137@tcp for MGC192.168.203.137@tcp not replayable, auto-deactivating 00000100:00080000:3.0:1789762997.560216:0:2397:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:3.0:1789762997.560220:0:2397:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff9360061dfb80 x1876702067578624/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560229:0:2397:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff936020dad880 x1876702067579520/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 0 dl 1789763002 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560242:0:2397:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff936010ef5880 x1876702067580416/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 0 dl 1789763007 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560251:0:2397:0:(client.c:3589:ptlrpc_abort_inflight()) @@@ inflight req@ffff936020dac700 x1876702067581312/t0(0) o400->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 224/224 e 0 to 0 dl 1789763011 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1789762997.560265:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1789762997.560269:0:2397:0:(pinger.c:193:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:3.0:1789762997.560271:0:2397:0:(mgc_request.c:1168:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1789762997.560275:0:2397:0:(import.c:418:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00000400:3.0:1789762997.560290:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061de680 x1876702067578880/t0(0) o400->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560312:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061de680 x1876702067578880/t0(0) o400->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762997.560319:0:2397:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789762997.560322:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff936020592000: Connection to lustre-MDT0001 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789762997.560325:0:2397:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1789762997.560330:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061dce00 x1876702067579136/t0(0) o400->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560355:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061dce00 x1876702067579136/t0(0) o400->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762997.560365:0:2397:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789762997.560368:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-OST0001-osc-ffff936020592000: Connection to lustre-OST0001 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789762997.560372:0:2397:0:(import.c:418:ptlrpc_pinger_force()) lustre-OST0001_UUID: waking up pinger s:DISCONN 00000100:00000400:3.0:1789762997.560383:0:2397:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762981/real 1789762981] req@ffff9360061dd180 x1876702067579392/t0(0) o400->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762997.560407:0:2397:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff9360061dd180 x1876702067579392/t0(0) o400->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 224/224 e 0 to 1 dl 1789762997 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762997.560412:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067582208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789762997.560425:0:2397:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1789762997.560431:0:2397:0:(import.c:182:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff936008b5a800: Connection to lustre-MDT0001 (at 192.168.203.137@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:3.0:1789762997.560432:0:2397:0:(import.c:418:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00100000:3.0:1789762997.560444:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:0607efc5-93ac-4424-8fa4-2e162e139892:2397:1876702067578624:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762997.560456:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067578880:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762997.560474:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2397:1876702067579136:192.168.203.137@tcp:400:kworker.0 00000100:00100000:3.0:1789762997.560487:0:2397:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2397:1876702067579392:192.168.203.137@tcp:400:kworker.0 00000100:00100000:2.0:1789762997.560521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789762997.560571:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789762997.560612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789762997.560647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067582720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789762997.560683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067582848:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789762998.313982:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762999.337910:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763000.361893:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763000.617949:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617958:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617964:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617969:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617973:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617976:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617979:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1789763000.617981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763000.617983:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:3.0:1789763001.385912:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763002.409967:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763002.665934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.665947:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.665956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.665968:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.665975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.665979:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.665988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.665991:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.665993:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.665997:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.666001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.666013:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763002.666021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763002.666034:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1789763003.433948:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763004.458254:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763005.481910:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763005.737936:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762995/real 1789762995] req@ffff936020dadc00 x1876702067581696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763005.737976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067581696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763005.737997:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763005.738024:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763005.738028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067581696:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789763005.738136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738146:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763005.738151:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738156:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763005.738161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738166:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763005.738170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763005.738181:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763005.738191:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 468 00000100:00080000:2.0:1789763005.738206:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763005.738214:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763005.738227:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763005.738263:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067582976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763005.738292:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738296:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763005.738299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738302:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763005.738305:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763005.738309:0:8:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1789763005.738337:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582976:192.168.203.137@tcp:8:kworker.0 00000080:00200000:2.0:1789763006.249987:0:37:0:(vvp_io.c:1826:vvp_io_init()) [0x240000402:0x1a:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1789763006.250003:0:37:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789763006.250029:0:37:0:(vvp_io.c:304:vvp_io_fini()) [0x240000402:0x1a:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00010000:00010000:2.0:1789763006.505902:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763007.529934:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763007.785889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef4a80 x1876702067582848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.785956:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067582848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.785972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef5180 x1876702067582592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.785988:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067582592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef4700 x1876702067582720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786015:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067582720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786027:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef5c00 x1876702067582336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067582336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef4000 x1876702067582464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067582464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786089:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789762997/real 1789762997] req@ffff936010ef4380 x1876702067582208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763007.786103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067582208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763007 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763007.786123:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763007.786145:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763007.786150:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067582848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763007.786156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763007.786163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763007.786167:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582592:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763007.786172:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763007.786177:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763007.786181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067582720:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763007.786186:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763007.786191:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763007.786195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582336:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763007.786202:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763007.786209:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763007.786213:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582464:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763007.786219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763007.786226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763007.786233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763007.786235:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763007.786241:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067582208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763007.786252:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786263:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786285:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763007.786317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067583104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763007.786342:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763007.786345:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786348:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786353:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763007.786354:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067583104:192.168.203.137@tcp:250:kworker.0 00000008:00100000:2.0:1789763007.786362:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763007.786378:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067583232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763007.786417:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763007.786422:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786427:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786440:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763007.786447:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763007.786462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763007.786476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067583360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786490:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763007.786492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763007.786495:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763007.786498:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786501:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786504:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763007.786509:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763007.786516:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763007.786519:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067583488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763007.786535:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763007.786539:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786545:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786550:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763007.786556:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763007.786561:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763007.786571:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067583616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763007.786589:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763007.786593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 470 00000100:00080000:2.0:1789763007.786597:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 5s 00000100:00080000:2.0:1789763007.786602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763007.786603:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067583616:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789763007.786607:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763007.786619:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067583744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763007.786632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763007.786635:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1789763007.786650:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067583744:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763008.553895:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763009.577922:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763010.601928:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763010.793858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793870:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793877:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793880:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793884:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793887:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793890:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793894:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793897:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763010.793899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763010.793901:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:2.0:1789763011.625902:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763012.649899:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763013.673933:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763014.699045:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763015.722032:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763016.746110:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763017.770084:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763018.794063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763019.818141:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763020.010006:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763005/real 1789763005] req@ffff9360061dc000 x1876702067582976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763020.010040:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067582976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763020.010086:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763020.010119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763020.010125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067582976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763020.010498:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583872:192.168.203.137@tcp:8:kworker.0 00010000:00010000:2.0:1789763021.866016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763022.057988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061dfb80 x1876702067583744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067583744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058061:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061de680 x1876702067583616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058073:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067583616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061dce00 x1876702067583488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067583488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061dd180 x1876702067583360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067583360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061dca80 x1876702067583232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058121:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067583232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058128:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763007/real 1789763007] req@ffff9360061df800 x1876702067583104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763022.058143:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067583104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763022 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763022.058167:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058200:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067583744:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763022.058208:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058215:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067583616:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763022.058225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583488:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763022.058275:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058294:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583360:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763022.058302:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058319:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583232:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763022.058324:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763022.058330:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763022.058333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067583104:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763022.058463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763022.058589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067584128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763022.058655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067584256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763022.058695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067584384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763022.058765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763022.058840:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584640:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763022.890127:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763024.938014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763025.962078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763028.009994:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763029.034083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763031.082025:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763032.106076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763034.154008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763035.178049:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:2.0:1789763035.881924:0:2386:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.137@tcp (192.168.203.137:988) timed out; the network or node may be down. 00010000:00010000:2.0:1789763037.225901:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763038.250104:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763040.298001:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763040.425956:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763020/real 1789763020] req@ffff936020dad500 x1876702067583872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763040 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763040.426032:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067583872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763040 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763040.426073:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763040.426109:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763040.426116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067583872:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789763040.426173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426191:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763040.426198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426204:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763040.426210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426216:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763040.426221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763040.426234:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763040.426247:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 492 00000100:00080000:2.0:1789763040.426283:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763040.426293:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763040.426309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763040.426349:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067584768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763040.426384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426400:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763040.426404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426408:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763040.426412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763040.426415:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:3.0:1789763040.426483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584768:192.168.203.137@tcp:8:kworker.0 00010000:00010000:3.0:1789763041.322001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763042.473877:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef6d80 x1876702067584512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067584512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473912:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef6300 x1876702067584640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473920:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067584640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473926:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef4e00 x1876702067584128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473933:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067584128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473939:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef5500 x1876702067584256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473946:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067584256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473952:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef7100 x1876702067584384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473959:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067584384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473964:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789763022/real 1789763022] req@ffff936010ef6680 x1876702067584000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763042.473972:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067584000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763042 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763042.473987:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474011:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584512:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763042.474015:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584640:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763042.474024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474027:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067584128:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763042.474032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474035:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474037:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067584256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763042.474040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474046:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067584384:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763042.474051:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763042.474054:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763042.474056:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584000:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789763042.474065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763042.474075:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474083:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00080000:2.0:1789763042.474099:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474107:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763042.474138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067584896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763042.474163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763042.474169:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474172:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00080000:2.0:1789763042.474176:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474181:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763042.474192:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.474204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067584896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763042.474207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067585024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763042.474220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763042.474225:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474228:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00080000:2.0:1789763042.474232:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474236:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763042.474240:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763042.474258:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067585152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763042.474268:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474272:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789763042.474275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763042.474280:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474284:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00080000:2.0:1789763042.474289:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474293:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763042.474300:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.474300:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763042.474316:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067585280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763042.474328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763042.474334:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585152:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763042.474335:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474344:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00080000:2.0:1789763042.474349:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474354:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763042.474361:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.474367:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763042.474375:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067585408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763042.474387:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763042.474393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763042.474396:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763042.474400:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 495 00000100:00100000:3.0:1789763042.474404:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067585408:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763042.474407:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789763042.474415:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763042.474421:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763042.474433:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067585536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763042.474481:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067585536:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763043.369995:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1789763043.498594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061dd500 x1876702067585536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.498645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067585536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.498668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.498677:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.498685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067585536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.498701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061df480 x1876702067585408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.498726:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067585408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.498744:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.498751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.498760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067585408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.498774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061dc700 x1876702067585280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.498887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067585280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.498903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.498911:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.498920:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763043.498934:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061dc380 x1876702067585024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.498960:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067585024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.498978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.498986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.498993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.499005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061dd880 x1876702067585152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.499030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067585152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.499046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.499053:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.499060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067585152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763043.499073:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763043] req@ffff9360061ddc00 x1876702067584896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.499094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067584896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.499107:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.499113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.499119:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067584896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763043.499130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763040/real 1789763043] req@ffff9360061de300 x1876702067584768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763043.499151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067584768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763043.499164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763043.499170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763043.499177:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067584768:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763045.481936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.481945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763045.481949:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.481960:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.481972:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.481979:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763045.482023:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067585664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763045.482059:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.482072:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482075:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482086:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763045.482097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067585792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763045.482114:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482117:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.482120:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763045.482140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067585920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763045.482161:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 513 00000100:00080000:3.0:1789763045.482169:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482173:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482178:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763045.482193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067586048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763045.482210:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482215:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.482221:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482227:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482234:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763045.482252:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067586176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763045.482276:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482284:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.482290:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482297:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482310:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763045.482328:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067586304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763045.482345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763045.482358:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763045.482361:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763045.482368:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 515 00000100:00080000:3.0:1789763045.482375:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763045.482380:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763045.482386:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763045.482406:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067586432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763046.441922:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763049.513941:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763050.538078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763050.793927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.793957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763050.793963:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.793988:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794005:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794016:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763050.794079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067586560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763050.794127:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794139:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794143:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794150:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794163:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763050.794179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067586688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763050.794202:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794214:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794223:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794232:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794238:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763050.794253:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067586816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763050.794273:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794279:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794282:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794287:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763050.794304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067586944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794321:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763050.794323:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794328:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794333:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794337:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794344:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763050.794357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067587072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763050.794382:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794388:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794391:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794396:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794402:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763050.794421:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067587200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763050.794439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763050.794443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763050.794446:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763050.794452:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 518 00000100:00080000:3.0:1789763050.794457:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763050.794461:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763050.794470:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763050.794495:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067587328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763052.585969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763053.610031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763055.657998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763055.913864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.913875:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763055.913878:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.913888:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.913898:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.913905:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763055.913937:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067587456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.913959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.913962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763055.913963:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.913967:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.913969:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.913971:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.913979:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763055.913988:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067587584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.913994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.913997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763055.913998:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.914002:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.914004:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.914007:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.914010:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763055.914016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067587712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.914023:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.914025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763055.914027:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.914030:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.914032:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.914038:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.914041:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763055.914052:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067587840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.914058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.914061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763055.914062:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.914065:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.914067:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.914069:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.914072:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763055.914079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067587968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.914085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.914088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763055.914089:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.914091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.914094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.914096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.914099:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763055.914107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067588096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763055.914115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763055.914118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763055.914119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763055.914123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 523 00000100:00080000:3.0:1789763055.914125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763055.914127:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763055.914130:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763055.914139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067588224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1789763056.681933:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763058.729910:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763059.754028:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763061.033937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.033950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763061.033957:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.033971:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.033985:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.033994:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763061.034043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067588352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763061.034093:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034098:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034106:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034119:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763061.034133:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067588480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763061.034151:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034156:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034159:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034164:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034169:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763061.034183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067588608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034206:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763061.034216:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034224:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034235:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034243:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034250:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763061.034266:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067588736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763061.034297:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034301:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034307:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034311:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763061.034329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067588864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763061.034348:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034352:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034357:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034363:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763061.034382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067588992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763061.034403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763061.034408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763061.034412:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763061.034419:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 528 00000100:00080000:3.0:1789763061.034424:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763061.034435:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763061.034450:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763061.034471:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067589120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763061.801969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763064.873974:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763066.153904:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.153921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763066.153929:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.153949:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.153969:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.153984:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763066.154060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067589248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763066.154128:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154137:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154145:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154154:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154173:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763066.154205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067589376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763066.154237:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154252:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154262:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154275:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154285:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763066.154317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067589504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763066.154352:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154359:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154369:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154388:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154402:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763066.154430:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067589632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154448:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763066.154465:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154481:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154494:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154503:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154514:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763066.154549:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067589760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763066.154641:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154657:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154692:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763066.154739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067589888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763066.154762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763066.154772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763066.154781:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763066.154878:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 533 00000100:00080000:3.0:1789763066.154894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763066.154910:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763066.154929:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763066.154967:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067590016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763067.946024:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763071.017996:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763071.273919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.273951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763071.273957:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.273973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.273989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274004:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763071.274052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067590144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763071.274096:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274102:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274106:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274110:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763071.274139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067590272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763071.274170:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274179:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274184:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274193:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274207:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763071.274222:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067590400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763071.274252:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274260:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274270:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274279:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274290:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763071.274322:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067590528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274343:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763071.274354:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274362:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274371:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274385:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274401:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763071.274441:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067590656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763071.274544:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274564:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274573:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274608:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274619:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763071.274648:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067590784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763071.274670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763071.274676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763071.274680:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763071.274687:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 539 00000100:00080000:3.0:1789763071.274696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763071.274712:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763071.274725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763071.274754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067590912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763074.089951:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763076.393924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.393952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763076.393956:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.393967:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.393988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.393996:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763076.394048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067591040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763076.394079:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394088:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394096:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763076.394105:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067591168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763076.394118:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394123:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394125:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394132:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394136:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763076.394144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067591296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763076.394155:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394157:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394163:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763076.394173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067591424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763076.394182:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394185:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394187:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394190:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394193:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763076.394200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067591552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394209:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763076.394210:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394212:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394215:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394218:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394222:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763076.394229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067591680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763076.394252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763076.394255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763076.394257:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763076.394262:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 544 00000100:00080000:2.0:1789763076.394267:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763076.394277:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763076.394302:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763076.394325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067591808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763077.161908:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763080.234010:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763081.514047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763081.514075:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514093:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514115:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514130:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763081.514204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067591936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763081.514269:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514275:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514281:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514287:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514302:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763081.514327:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067592064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763081.514365:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514379:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514391:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514399:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514411:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763081.514439:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067592192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514467:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763081.514471:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514478:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514486:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514526:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514543:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763081.514570:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067592320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763081.514601:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514617:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514631:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514646:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514663:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763081.514694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067592448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763081.514756:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514787:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514864:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514876:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763081.514906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067592576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763081.514932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763081.514940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763081.514945:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763081.514954:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 549 00000100:00080000:3.0:1789763081.514963:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763081.514979:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763081.514993:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763081.515025:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067592704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763083.305952:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763086.378007:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763086.633949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.633963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763086.633968:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634013:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634029:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634039:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763086.634134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067592832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763086.634228:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634234:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634241:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634247:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763086.634304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067592960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763086.634343:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634350:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634357:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634363:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634372:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763086.634390:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067593088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634404:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634411:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763086.634417:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634448:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634455:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634459:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634488:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763086.634503:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067593216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763086.634518:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634523:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634527:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634536:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634546:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763086.634567:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067593344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763086.634608:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634625:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634633:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763086.634652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067593472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763086.634683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763086.634691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763086.634694:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763086.634702:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 554 00000100:00080000:3.0:1789763086.634711:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763086.634722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763086.634732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763086.634782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067593600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763089.450001:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763091.753943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.753957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763091.753963:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.753978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.753994:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754006:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763091.754087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067593728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763091.754163:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754172:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754184:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754199:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763091.754221:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067593856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763091.754246:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754258:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754263:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754276:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754284:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763091.754300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067593984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763091.754323:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754328:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754336:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754342:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754348:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763091.754363:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067594112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763091.754393:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754401:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754407:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754412:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754419:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763091.754467:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067594240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763091.754493:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754506:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754517:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754530:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754543:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763091.754568:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067594368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763091.754635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763091.754642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763091.754648:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763091.754662:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 559 00000100:00080000:2.0:1789763091.754670:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763091.754684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763091.754691:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763091.754711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067594496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763092.522063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763095.593970:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763096.874072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763096.874092:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874106:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874121:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874131:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763096.874180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067594624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763096.874232:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874245:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874249:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874254:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874269:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763096.874286:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067594752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763096.874314:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874342:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874347:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874352:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763096.874377:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067594880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763096.874423:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874429:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874434:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874443:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874453:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763096.874476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067595008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763096.874508:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874523:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874536:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874544:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874556:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763096.874587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067595136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763096.874630:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874643:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874660:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874672:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874683:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763096.874712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067595264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763096.874738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763096.874746:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763096.874753:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763096.874763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 564 00000100:00080000:3.0:1789763096.874773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763096.874782:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763096.874794:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763096.874894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067595392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763098.665990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763101.738010:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763104.809958:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763107.881898:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763110.953993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763114.025999:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763117.097962:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763120.170038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763122.474013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763122.474034:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474051:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474069:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474082:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763122.474145:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067599104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474190:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763122.474222:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474231:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474237:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474256:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474273:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763122.474304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067599232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474330:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763122.474342:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474351:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474360:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474370:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474381:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763122.474417:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067599360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474431:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763122.474444:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474456:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474464:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474471:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474479:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763122.474502:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067599488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763122.474531:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474544:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474560:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474576:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474592:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763122.474625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067599616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763122.474684:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474697:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474706:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474717:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474729:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763122.474760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067599744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763122.474854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763122.474864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763122.474869:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763122.474878:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 590 00000100:00080000:3.0:1789763122.474886:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763122.474894:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763122.474911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763122.474950:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067599872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763123.242015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763126.313969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763127.593960:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.593974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763127.593981:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.593996:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763127.594091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067600000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763127.594144:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594149:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594153:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594207:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763127.594231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067600128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763127.594284:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594299:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594308:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594318:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594331:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763127.594360:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067600256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763127.594406:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594415:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594424:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594433:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594444:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763127.594496:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067600384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763127.594611:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594630:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594639:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594651:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594666:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763127.594689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067600512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763127.594728:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594753:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594759:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763127.594779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067600640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763127.594858:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763127.594867:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763127.594872:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763127.594883:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 595 00000100:00080000:3.0:1789763127.594889:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763127.594894:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763127.594914:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763127.594961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067600768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763129.385985:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763132.457954:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763135.529949:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763135.786123:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763135.786175:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763135.786197:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786244:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786253:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786261:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786267:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786276:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763135.786283:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:3.0:1789763137.834011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763137.834032:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834047:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834065:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834076:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763137.834166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067601792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763137.834220:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834225:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834236:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834252:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763137.834268:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067601920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763137.834291:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834296:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834300:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834312:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834323:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763137.834351:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067602048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834374:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763137.834385:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834393:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834403:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834411:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834422:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763137.834453:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067602176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763137.834493:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834501:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834510:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834518:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834530:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763137.834564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067602304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763137.834633:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834655:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834670:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763137.834737:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067602432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763137.834775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763137.834783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763137.834788:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763137.834912:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 605 00000100:00080000:3.0:1789763137.834927:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763137.834949:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763137.834970:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763137.835008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067602560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763138.601994:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763141.674041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763142.953890:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.953908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763142.953914:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.953928:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.953945:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.953956:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763142.954018:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067602688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763142.954089:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954094:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954104:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954119:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763142.954138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067602816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763142.954165:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954179:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954184:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954189:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954196:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763142.954213:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067602944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763142.954230:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954235:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954241:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954246:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954251:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763142.954263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067603072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763142.954280:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954284:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954288:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954297:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954306:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763142.954322:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067603200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763142.954341:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954346:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954351:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954356:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954362:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763142.954379:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067603328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763142.954429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763142.954445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763142.954452:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763142.954461:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 610 00000100:00080000:3.0:1789763142.954471:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763142.954489:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763142.954502:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763142.954535:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067603456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763144.745985:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763147.817988:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763148.073927:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.073939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763148.073943:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.073955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.073966:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.073977:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763148.074045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067603584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763148.074110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074117:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074132:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074147:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763148.074172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067603712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763148.074201:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074207:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074213:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074232:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074241:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763148.074281:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067603840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074301:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074308:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763148.074312:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074332:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074363:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074374:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763148.074395:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067603968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074427:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763148.074436:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074443:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074449:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074461:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074470:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763148.074494:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067604096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763148.074521:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074527:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074533:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074542:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074552:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763148.074568:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067604224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763148.074612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763148.074621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763148.074628:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763148.074634:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 615 00000100:00080000:3.0:1789763148.074638:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763148.074641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763148.074646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763148.074659:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067604352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763150.890014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763153.193930:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.193946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763153.193951:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.193965:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194002:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763153.194065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067604480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763153.194112:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194118:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194127:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194142:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763153.194160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067604608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763153.194191:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194205:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194210:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194215:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194222:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763153.194237:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067604736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763153.194257:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194262:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194280:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194286:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194291:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763153.194308:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067604864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763153.194332:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194338:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194344:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194348:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194354:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763153.194368:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067604992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194380:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763153.194386:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194391:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194399:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194408:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194418:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763153.194440:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067605120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763153.194453:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763153.194466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763153.194470:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763153.194478:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 621 00000100:00080000:3.0:1789763153.194485:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763153.194501:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763153.194522:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763153.194556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067605248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763153.961992:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763157.034020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763158.313928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.313988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763158.313993:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314006:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314019:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763158.314073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067605376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763158.314117:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314124:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314136:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314150:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763158.314171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067605504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763158.314197:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314203:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314208:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314215:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314222:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763158.314236:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067605632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763158.314256:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314261:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314266:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314271:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314277:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763158.314291:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067605760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763158.314312:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314317:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314322:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314328:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314335:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763158.314350:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067605888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314374:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763158.314384:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314390:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314397:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314405:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314413:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763158.314428:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067606016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763158.314443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763158.314448:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763158.314451:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763158.314458:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 626 00000100:00080000:3.0:1789763158.314463:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763158.314469:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763158.314476:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763158.314492:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067606144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763160.106012:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763163.178083:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763163.433966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.433980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763163.433986:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.433999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434011:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763163.434077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067606272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763163.434121:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434126:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434134:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434147:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763163.434163:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067606400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763163.434192:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434206:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434210:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434215:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434221:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763163.434234:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067606528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763163.434252:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434256:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434262:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434267:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434273:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763163.434289:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067606656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763163.434310:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434316:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434320:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434324:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434330:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763163.434346:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067606784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763163.434383:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434393:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434398:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434409:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434425:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763163.434454:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067606912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763163.434493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763163.434502:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763163.434507:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763163.434518:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 631 00000100:00080000:3.0:1789763163.434529:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763163.434541:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763163.434549:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763163.434576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067607040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763166.249981:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763168.553964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.553983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763168.553991:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554011:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554031:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554047:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763168.554114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067607168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763168.554170:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554178:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554187:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554196:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554217:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763168.554249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067607296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763168.554282:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554288:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554294:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554300:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763168.554326:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067607424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763168.554358:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554365:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554371:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554377:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554383:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763168.554397:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067607552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763168.554416:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554422:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554427:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554432:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554439:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763168.554458:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067607680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763168.554535:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554556:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554568:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554588:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554608:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763168.554631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067607808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763168.554650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763168.554660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763168.554662:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763168.554668:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 636 00000100:00080000:3.0:1789763168.554673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763168.554680:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763168.554690:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763168.554708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067607936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763169.322014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763172.394041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763173.674416:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763173.674422:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763173.674429:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 641 00000100:00080000:0.0:1789763173.674432:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763173.674435:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763173.674440:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763173.674453:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067608832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763175.465990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763178.537971:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763178.793940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.793953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763178.793959:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.793974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.793988:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.793997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763178.794048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067608960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763178.794105:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794110:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794114:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794131:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763178.794150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067609088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763178.794172:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794181:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794186:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794199:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794208:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763178.794223:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067609216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763178.794244:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794249:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794254:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794259:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794264:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763178.794276:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067609344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763178.794297:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794307:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794314:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794319:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794325:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763178.794338:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067609472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763178.794356:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794362:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794366:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794370:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794375:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763178.794389:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067609600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763178.794400:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763178.794407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763178.794413:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763178.794427:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 646 00000100:00080000:3.0:1789763178.794436:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763178.794447:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763178.794459:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763178.794483:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067609728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763181.610044:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763183.914012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763183.914032:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914047:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914064:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914075:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763183.914140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067609856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763183.914194:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914199:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914203:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914207:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914221:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763183.914235:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067609984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914262:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763183.914264:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914269:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914272:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914278:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914283:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763183.914296:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067610112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763183.914315:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914324:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914328:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914334:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763183.914354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067610240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914385:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763183.914391:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914396:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914400:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914404:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914410:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763183.914427:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067610368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763183.914445:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914449:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914480:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914485:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914491:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763183.914510:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067610496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763183.914525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763183.914530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763183.914532:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763183.914538:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 651 00000100:00080000:0.0:1789763183.914544:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763183.914548:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763183.914553:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763183.914568:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067610624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763184.682055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763187.753997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763189.033937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.033953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763189.033959:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.033974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.033989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.033998:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763189.034050:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067610752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763189.034098:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034112:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034124:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763189.034138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067610880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763189.034157:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034162:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034166:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034171:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034177:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763189.034192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067611008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034209:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763189.034211:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034216:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034219:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034225:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034238:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763189.034261:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067611136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034277:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763189.034285:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034293:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034300:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034305:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034311:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763189.034329:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067611264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763189.034347:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034354:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034359:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034365:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034371:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763189.034395:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067611392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763189.034409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763189.034414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763189.034417:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763189.034424:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 656 00000100:00080000:0.0:1789763189.034440:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763189.034445:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763189.034450:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763189.034465:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067611520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789763190.826003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763193.898023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763194.153885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.153898:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763194.153905:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.153919:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.153936:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.153947:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763194.154013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067611648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763194.154092:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154108:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154114:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154130:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763194.154162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067611776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763194.154187:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154193:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154198:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154210:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154223:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763194.154240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067611904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763194.154262:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154268:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154273:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154279:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154285:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763194.154300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067612032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763194.154320:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154331:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154343:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154349:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154356:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763194.154372:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067612160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763194.154392:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154397:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154402:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154414:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154428:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763194.154454:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067612288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763194.154529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763194.154544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763194.154551:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763194.154567:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 661 00000100:00080000:0.0:1789763194.154572:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763194.154582:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763194.154590:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763194.154613:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067612416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763194.154713:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067611776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.154733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.154738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.154742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067611776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763194.154777:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067611904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763194.154842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763194.154860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763194/real 1789763194] req@ffff936020dac700 x1876702067611904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763194.154875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067611904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.154886:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.154890:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.154894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067611904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763194.154901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763194.154925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763194.154937:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763194/real 1789763194] req@ffff936020dad880 x1876702067612032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763194.154951:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067612032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.154961:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.154964:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.154969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763194.154976:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763194.154997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763194.155009:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763194/real 1789763194] req@ffff936020daca80 x1876702067612160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763194.155023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067612160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.155032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.155036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.155039:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763194.155046:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067612288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763194.155066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763194.155079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763194/real 1789763194] req@ffff936020dac000 x1876702067612288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763194.155093:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067612288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.155102:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.155105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.155109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067612288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763194.155115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067612416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763194.155135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763194.155148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763194/real 1789763194] req@ffff936020dafb80 x1876702067612416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763194.155162:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067612416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763194.155172:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763194.155175:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763194.155179:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067612416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763194.922082:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763195.945993:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763196.969971:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763197.994054:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763199.018069:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763199.273947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.273963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763199.273970:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.273989:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763199.274080:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067612544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763199.274142:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763199.274155:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.274163:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274171:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274181:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763199.274201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763199.274221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067612544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763199.274229:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067612672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763199.274248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763199.274256:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.274262:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274270:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274280:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763199.274289:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763199.274312:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067612800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763199.274333:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274341:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763199.274346:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.274354:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274364:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274373:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763199.274382:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763199.274390:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763199.274408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067612928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763199.274425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763199.274433:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.274437:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274443:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274448:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763199.274456:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612800:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789763199.274460:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763199.274487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067613056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763199.274501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274506:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763199.274513:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612928:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789763199.274513:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763199.274529:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274536:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274542:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763199.274549:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763199.274567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067613184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763199.274576:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613056:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789763199.274604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763199.274610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763199.274613:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763199.274625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067613184:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763199.274627:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 667 00000100:00080000:2.0:1789763199.274664:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763199.274674:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763199.274686:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763199.274715:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067613312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763199.274848:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067613312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763200.041961:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763201.066067:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763202.090038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763202.346173:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789763202.346235:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763202.346262:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763202.346325:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763202.346344:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763202.346368:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763202.346387:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763202.346419:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763202.346429:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061dce00 x1876702067612928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789763202.346445:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763202.346511:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067612928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.346539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.346548:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.346565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763202.346579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061dd180 x1876702067613056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.346607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067613056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.346657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.346665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.346672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763202.346685:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061dc000 x1876702067612672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.346720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067612672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.346736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.346742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.346749:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763202.346762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061de680 x1876702067612800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.346872:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067612800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.346894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.346903:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.346913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067612800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763202.346932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061dfb80 x1876702067612544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.346960:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067612544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.346977:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.346984:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.346991:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067612544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763202.347040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061df800 x1876702067613312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.347065:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067613312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.347082:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.347089:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.347096:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067613312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763202.347110:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763199/real 1789763202] req@ffff9360061dca80 x1876702067613184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763202.347131:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067613184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763202.347147:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763202.347153:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763202.347159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067613184:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789763203.113958:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763204.138012:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763204.393920:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763204.393948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763204.393953:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763204.393968:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 672 00000100:00080000:2.0:1789763204.393982:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.393992:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763204.394039:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067613440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763204.394078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763204.394083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763204.394085:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763204.394091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 672 00000100:00080000:2.0:1789763204.394095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.394100:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763204.394113:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763204.394130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067613568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763204.394146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763204.394151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763204.394156:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763204.394160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067613440:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763204.394166:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 672 00000100:00080000:2.0:1789763204.394179:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.394183:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763204.394190:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763204.394215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067613696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763204.394235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763204.394243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763204.394247:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763204.394255:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 672 00000800:00000100:1.0:1789763204.394265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763204.394269:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.394284:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763204.394295:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763204.394321:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067613824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763204.394329:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061dd500 x1876702067613440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763204.394374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763204.394383:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067613440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763204.394387:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763204.394395:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763204.394415:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 672 00000100:00080000:1.0:1789763204.394424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.394438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763204.394446:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763204.394448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067613440:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763204.394466:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763204.394481:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763204.394505:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763204.394515:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067613952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763204.394542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763204.394549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763204.394553:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763204.394561:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 672 00000800:00000100:1.0:1789763204.394571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763204.394577:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.394592:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763204.394634:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061df480 x1876702067613568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763204.394635:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763204.394686:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067614080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763204.394687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067613568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.394751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763204.394754:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763204.394768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763204.394770:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763204.394780:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763204.394787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763204.394865:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613696:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763204.394876:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 672 00000100:00080000:2.0:1789763204.394890:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763204.394903:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763204.394917:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763204.394919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763204.394957:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067614208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763204.394961:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061df100 x1876702067613696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763204.395030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067613696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.395050:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.395057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763204.395063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763204.395088:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763204.395136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763204.395161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061dd880 x1876702067613824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763204.395183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067613824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.395198:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.395203:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763204.395209:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763204.395241:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763204.395284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763204.395308:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061ddc00 x1876702067613952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763204.395334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067613952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.395351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.395358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763204.395366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067613952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763204.395382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763204.395425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763204.395450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061de300 x1876702067614080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763204.395474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067614080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.395490:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.395497:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763204.395504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763204.395517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763204.395559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763204.395585:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763204/real 1789763204] req@ffff9360061ddf80 x1876702067614208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763204.395641:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067614208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763204.395672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763204.395679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763204.395686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763205.162029:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763206.186027:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763207.210034:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763208.234076:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763209.258066:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763209.514026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763209.514046:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514060:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763209.514084:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763209.514152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067614336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763209.514189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763209.514197:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514202:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514206:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763209.514227:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763209.514239:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763209.514266:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067614464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763209.514279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763209.514287:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514293:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514314:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763209.514322:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763209.514330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067614336:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763209.514331:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763209.514361:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067614592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763209.514392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514397:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763209.514399:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514403:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514407:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763209.514412:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763209.514418:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763209.514432:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067614720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763209.514443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763209.514449:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514458:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 677 00000800:00000100:1.0:1789763209.514461:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763209.514463:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763209.514471:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763209.514477:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763209.514492:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067614848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763209.514518:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763209.514524:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763209.514528:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514532:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763209.514537:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa5180 x1876702067614336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763209.514541:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763209.514551:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763209.514619:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067614976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763209.514624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067614336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763209.514650:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763209.514657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763209.514660:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763209.514661:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763209.514697:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763209.514698:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 677 00000100:00080000:3.0:1789763209.514710:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763209.514713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067614336:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763209.514722:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763209.514732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763209.514735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763209.514752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067615104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763209.514799:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.514823:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa4380 x1876702067614464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.514877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067614464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.514896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.514911:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.514915:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763209.514944:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763209.514969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.514983:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa7800 x1876702067614592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.515014:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067614592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.515025:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.515029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.515033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763209.515040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763209.515062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.515074:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa4a80 x1876702067614720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.515088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067614720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.515098:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.515101:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.515106:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763209.515113:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763209.515132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.515144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa7b80 x1876702067614848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.515158:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067614848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.515167:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.515170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.515174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067614848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763209.515180:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763209.515206:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.515247:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa7100 x1876702067614976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.515268:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067614976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.515285:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.515291:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.515298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067614976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763209.515328:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067615104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763209.515363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763209.515380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763209/real 1789763209] req@ffff936010aa5500 x1876702067615104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763209.515395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067615104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763209.515405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763209.515409:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763209.515413:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067615104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763210.282018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763211.306001:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763212.330067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763213.354087:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763214.378128:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763214.633949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.633967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763214.633974:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.633990:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763214.634018:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763214.634066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067615232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763214.634103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.634108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763214.634110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634116:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634121:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763214.634125:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763214.634136:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763214.634158:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067615360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763214.634171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.634175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763214.634188:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634196:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634203:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763214.634212:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763214.634222:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067615232:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763214.634223:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763214.634259:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067615488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763214.634283:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.634289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763214.634291:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634296:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634303:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763214.634308:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763214.634316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789763214.634317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763214.634340:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067615616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763214.634354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.634359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763214.634362:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634367:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634373:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763214.634378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020daf480 x1876702067615232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763214.634383:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763214.634395:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763214.634425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067615232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763214.634426:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067615744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763214.634465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763214.634470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763214.634471:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634475:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634483:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763214.634493:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763214.634496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763214.634498:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763214.634510:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763214.634518:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067615872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763214.634550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067615232:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763214.634574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763214.634574:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615360:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763214.634591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763214.634595:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763214.634604:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 682 00000100:00080000:0.0:1789763214.634617:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763214.634625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763214.634634:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763214.634649:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763214.634652:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020daed80 x1876702067615360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.634675:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067615360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763214.634684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067616000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763214.634690:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.634704:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.634718:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763214.634758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763214.634848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763214.634882:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020dae680 x1876702067615488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.634903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067615488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763214.634914:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.634918:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.634922:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763214.634929:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763214.634957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763214.634970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020dac380 x1876702067615616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.634984:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067615616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763214.634994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.634998:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.635001:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763214.635007:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763214.635026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763214.635037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020dafb80 x1876702067615744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.635049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067615744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763214.635059:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.635062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.635065:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067615744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763214.635070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067615872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763214.635089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763214.635100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020dac000 x1876702067615872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.635112:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067615872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763214.635121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.635123:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.635128:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067615872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763214.635134:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763214.635152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763214.635163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763214/real 1789763214] req@ffff936020daca80 x1876702067616000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763214.635175:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067616000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763214.635190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763214.635196:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763214.635202:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763215.402015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763216.426133:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763217.450018:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763218.474133:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763219.498051:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763219.753980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.753994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763219.754002:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754017:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754035:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754046:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763219.754122:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067616128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763219.754179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.754187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763219.754190:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754199:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754207:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763219.754234:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763219.754264:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067616256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763219.754279:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067616128:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763219.754304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.754310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763219.754313:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754321:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754329:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754335:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763219.754344:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763219.754367:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067616384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763219.754380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.754385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763219.754387:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754393:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 687 00000800:00000100:1.0:1789763219.754403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763219.754405:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754414:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763219.754423:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763219.754440:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067616512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763219.754451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.754457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763219.754460:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754466:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754471:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754482:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763219.754516:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061dc700 x1876702067616128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763219.754516:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763219.754539:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067616640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763219.754565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763219.754572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067616128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763219.754573:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763219.754581:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754595:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754608:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763219.754619:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763219.754623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789763219.754625:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763219.754635:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763219.754648:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067616768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763219.754650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067616128:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763219.754677:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763219.754682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763219.754685:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763219.754692:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 687 00000100:00080000:2.0:1789763219.754700:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763219.754712:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763219.754713:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763219.754730:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763219.754762:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067616896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763219.754788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.754877:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061dc380 x1876702067616256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.754898:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067616256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.754910:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.754915:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.754919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763219.754928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763219.754975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.754991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061ddf80 x1876702067616384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.755005:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067616384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.755016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.755020:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.755024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763219.755031:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763219.755055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.755068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061de300 x1876702067616512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.755082:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067616512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.755103:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.755106:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.755111:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763219.755118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763219.755155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.755180:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061ddc00 x1876702067616640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.755205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067616640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.755222:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.755229:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.755234:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067616640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763219.755243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763219.755273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.755288:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061dd880 x1876702067616768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.755302:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067616768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.755312:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.755316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.755320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763219.755346:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763219.755369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763219.755381:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763219/real 1789763219] req@ffff9360061df100 x1876702067616896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763219.755395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067616896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763219.755405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763219.755408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763219.755412:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067616896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763220.522105:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763221.546063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763222.569965:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763223.594049:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763224.618071:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763224.873947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.873961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763224.873968:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.873983:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874002:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874014:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763224.874092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067617024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763224.874148:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.874157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763224.874161:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.874169:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874177:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763224.874222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067617152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763224.874238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067617024:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763224.874244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.874258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763224.874262:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.874268:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874272:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874277:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874282:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763224.874301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067617280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763224.874314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.874320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763224.874323:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.874328:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874333:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874337:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874343:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763224.874358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067617408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763224.874369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763224.874380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.874385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763224.874388:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.874395:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874400:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874405:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874411:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763224.874425:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067617536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763224.874437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763224.874441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763224.874445:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763224.874481:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061df480 x1876702067617024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763224.874491:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874516:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874526:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763224.874532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067617024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763224.874550:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067617664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763224.874556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.874573:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763224.874581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763224.874584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067617024:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763224.874587:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763224.874595:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763224.874605:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 692 00000100:00080000:2.0:1789763224.874609:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763224.874615:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763224.874620:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763224.874633:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067617792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763224.874637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763224.874694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.874716:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061dd500 x1876702067617152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.874732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067617152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.874743:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.874747:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.874752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763224.874759:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763224.874781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.874794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061dca80 x1876702067617280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.874869:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067617280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.874881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.874885:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.874889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763224.874898:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763224.874925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.874940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061df800 x1876702067617408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.874954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067617408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.874964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.874968:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.874972:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763224.874978:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763224.874999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.875018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061dfb80 x1876702067617536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.875041:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067617536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.875057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.875063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.875070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067617536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763224.875082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067617664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763224.875123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.875140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061de680 x1876702067617664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.875156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067617664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.875167:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.875172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.875179:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067617664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763224.875216:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067617792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763224.875258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763224.875281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763224/real 1789763224] req@ffff9360061dc000 x1876702067617792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763224.875297:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067617792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763224.875308:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763224.875312:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763224.875316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067617792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763225.642086:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763226.666098:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763227.689970:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763228.714095:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763229.738015:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763229.994033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763229.994053:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994067:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994082:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994092:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763229.994142:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067617920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763229.994184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763229.994192:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994197:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994201:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994205:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994217:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763229.994233:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067618048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763229.994262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763229.994269:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994275:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994279:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994284:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994291:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763229.994303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067618176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763229.994325:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763229.994332:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994337:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994340:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994345:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994351:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763229.994361:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067618304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763229.994382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067617920:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763229.994384:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763229.994443:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994464:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994470:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994479:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994490:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763229.994530:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067618432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763229.994551:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763229.994561:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994569:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994578:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763229.994587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763229.994589:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763229.994625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067618560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763229.994638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763229.994643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763229.994647:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763229.994656:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 697 00000100:00080000:3.0:1789763229.994664:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763229.994669:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763229.994679:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763229.994680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa5880 x1876702067617920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763229.994707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067618688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763229.994727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067617920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.994749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.994760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.994767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067617920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763229.994779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763229.994827:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.994935:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa7480 x1876702067618048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.994952:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067618048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.994962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.994966:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.994971:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763229.994979:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763229.995008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.995020:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa5c00 x1876702067618176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.995034:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067618176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.995051:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.995055:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.995059:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763229.995097:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763229.995120:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.995132:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa6d80 x1876702067618304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.995144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067618304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.995154:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.995157:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.995160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763229.995168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763229.995187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.995198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa4000 x1876702067618432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.995213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067618432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.995227:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.995233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.995240:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763229.995267:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067618560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763229.995307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.995332:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa6300 x1876702067618560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.995351:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067618560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.995362:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.995365:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.995369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067618560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763229.995377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067618688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763229.995405:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763229.995449:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763229/real 1789763229] req@ffff936010aa6a00 x1876702067618688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763229.995464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067618688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763229.995474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763229.995478:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763229.995482:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067618688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763230.762091:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763231.786102:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763232.810081:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763233.834073:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763234.858114:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763235.114015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763235.114040:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763235.114057:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114086:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763235.114149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067618816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763235.114200:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763235.114205:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114211:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114215:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763235.114228:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763235.114244:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067618944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763235.114269:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763235.114277:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114285:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114289:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763235.114296:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763235.114310:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067619072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114326:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763235.114343:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763235.114345:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067618816:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763235.114364:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114399:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114407:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763235.114417:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763235.114442:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067619200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763235.114462:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763235.114471:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114495:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114501:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763235.114510:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763235.114512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763235.114539:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067619328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114562:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763235.114564:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763235.114569:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114575:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114581:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763235.114588:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763235.114602:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020dad880 x1876702067618816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763235.114603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067619456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763235.114722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763235.114729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763235.114733:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763235.114743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067618816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763235.114753:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 702 00000100:00080000:0.0:1789763235.114773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763235.114793:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763235.114884:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763235.114890:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763235.114899:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.114913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067618816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763235.114917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067619584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763235.114945:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763235.114996:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020dac700 x1876702067618944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115036:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067618944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115052:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115056:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067618944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763235.115093:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763235.115120:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115133:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020daf100 x1876702067619072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115147:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067619072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115157:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115160:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115164:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763235.115172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763235.115194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020dad500 x1876702067619200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115220:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067619200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763235.115244:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763235.115263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115275:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020dadf80 x1876702067619328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115298:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067619328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115308:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115311:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115314:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763235.115321:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067619456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763235.115342:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115354:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020daf800 x1876702067619456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067619456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115412:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067619456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763235.115420:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067619584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763235.115445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763235.115460:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763235/real 1789763235] req@ffff936020dae300 x1876702067619584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763235.115474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067619584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763235.115491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763235.115495:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763235.115499:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067619584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763235.882180:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763236.906130:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763237.930145:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763238.954123:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763239.978117:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763240.233980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.233995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763240.234003:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234018:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234036:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234048:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763240.234114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067619712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763240.234159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763240.234167:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234172:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234176:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234181:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763240.234210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067619840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763240.234225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763240.234233:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234240:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234243:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234248:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234253:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763240.234269:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067619968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763240.234293:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067619712:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763240.234298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763240.234324:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234331:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234359:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234371:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234379:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763240.234405:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067620096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763240.234420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763240.234429:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234433:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234461:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234466:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234471:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763240.234486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763240.234505:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067620224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763240.234530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763240.234539:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763240.234550:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00080000:0.0:1789763240.234557:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763240.234566:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020dad180 x1876702067619712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763240.234572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234591:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763240.234633:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067620352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763240.234636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067619712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763240.234717:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763240.234726:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763240.234726:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763240.234740:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763240.234747:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763240.234759:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 708 00000100:00100000:1.0:1789763240.234770:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067619712:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763240.234772:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763240.234792:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763240.234863:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763240.234897:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067620480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763240.234914:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763240.234979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020daea00 x1876702067619840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235019:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067619840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763240.235082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763240.235123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235147:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020dace00 x1876702067619968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235174:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067619968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235199:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067619968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763240.235208:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763240.235239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235254:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020dadc00 x1876702067620096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235269:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067620096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235282:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235296:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763240.235308:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763240.235372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020dae300 x1876702067620224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067620224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763240.235446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067620352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763240.235475:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235490:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020daf800 x1876702067620352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235505:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067620352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235514:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235518:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067620352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763240.235528:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067620480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763240.235549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763240.235562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763240/real 1789763240] req@ffff936020dadf80 x1876702067620480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763240.235576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067620480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763240.235586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763240.235589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763240.235593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067620480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763241.002084:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763242.026040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763243.050078:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763244.074113:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763245.097971:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763245.353917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.353936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763245.353943:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763245.353959:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.353975:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.353987:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763245.354062:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067620608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763245.354096:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763245.354105:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763245.354110:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354114:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763245.354151:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763245.354170:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067620736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763245.354184:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067620608:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763245.354193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763245.354202:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763245.354209:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354215:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354220:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763245.354226:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763245.354243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067620864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763245.354255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763245.354263:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763245.354267:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354272:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354278:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763245.354285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789763245.354290:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763245.354359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067620992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763245.354374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763245.354383:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763245.354389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020dad500 x1876702067620608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763245.354395:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354427:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354438:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067620608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763245.354451:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763245.354459:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.354465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763245.354477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067621120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763245.354487:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067620608:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763245.354493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763245.354502:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763245.354509:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620736:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789763245.354539:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763245.354562:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067621248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763245.354572:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763245.354588:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763245.354593:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763245.354596:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763245.354601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020daf100 x1876702067620736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763245.354607:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 713 00000100:00080000:0.0:1789763245.354618:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763245.354627:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067620736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763245.354637:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763245.354655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763245.354659:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067621376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763245.354665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.354677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763245.354692:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763245.354719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354731:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020dac700 x1876702067620864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763245.354744:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067620864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763245.354770:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.354774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.354777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763245.354782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763245.354800:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354809:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020dad880 x1876702067620992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763245.354875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067620992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763245.354885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.354889:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.354892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067620992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763245.354908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763245.354933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763245.354946:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020daca80 x1876702067621120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763245.354958:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067621120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763245.354971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.354977:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.354982:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763245.355011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067621248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763245.355048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763245.355082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020dac000 x1876702067621248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763245.355102:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067621248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763245.355117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.355140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.355147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067621248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763245.355160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067621376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763245.355197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763245.355213:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763245/real 1789763245] req@ffff936020dafb80 x1876702067621376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763245.355229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067621376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763245.355239:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763245.355243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763245.355247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067621376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763246.122063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763247.146102:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763248.170011:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763249.194032:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763250.218120:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763250.473975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.473988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763250.473993:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474034:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474044:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763250.474093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067621504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763250.474130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.474137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763250.474141:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474148:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474157:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474165:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763250.474183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763250.474206:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067621632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763250.474207:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067621504:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763250.474228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.474234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763250.474237:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474244:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474251:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474286:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763250.474296:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763250.474318:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067621760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763250.474328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763250.474337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.474343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763250.474346:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474352:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474359:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474366:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763250.474374:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763250.474392:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067621888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763250.474394:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061dd180 x1876702067621504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763250.474425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763250.474426:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067621504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763250.474432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763250.474435:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474442:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:1.0:1789763250.474447:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763250.474447:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474454:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763250.474455:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:2.0:1789763250.474462:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763250.474463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067621504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763250.474477:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067622016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763250.474492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.474497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763250.474501:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763250.474504:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621632:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763250.474507:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474513:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474520:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763250.474528:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763250.474545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067622144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763250.474548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763250.474564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763250.474571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763250.474572:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061dce00 x1876702067621632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763250.474575:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763250.474582:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 718 00000100:00080000:2.0:1789763250.474589:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763250.474597:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763250.474598:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067621632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763250.474605:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763250.474622:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067622272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763250.474635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.474642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.474649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763250.474661:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763250.474702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763250.474728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061dc000 x1876702067621760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763250.474750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067621760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763250.474762:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.474766:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.474771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763250.474779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763250.474852:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763250.474869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061de680 x1876702067621888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763250.474885:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067621888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763250.474896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.474900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.474904:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067621888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763250.474910:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763250.474933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763250.474945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061dfb80 x1876702067622016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763250.474959:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067622016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763250.474969:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.474973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.474976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763250.474999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067622144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763250.475022:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763250.475035:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061df800 x1876702067622144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763250.475049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067622144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763250.475068:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.475072:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.475076:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067622144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763250.475082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067622272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763250.475101:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763250.475113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763250/real 1789763250] req@ffff9360061dca80 x1876702067622272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763250.475127:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067622272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763250.475137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763250.475140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763250.475144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067622272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763251.242006:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763252.266059:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763253.290091:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763254.314100:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763255.338030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763255.593980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.593994:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763255.593999:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594013:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594027:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594036:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763255.594087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067622400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763255.594125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.594131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763255.594134:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594143:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594147:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763255.594159:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763255.594173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067622528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763255.594186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.594190:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763255.594193:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594198:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594202:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594208:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763255.594214:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763255.594258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067622656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763255.594272:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067622400:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763255.594279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.594288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763255.594291:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594297:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594303:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594309:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763255.594317:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763255.594334:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067622784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763255.594346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.594350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763255.594353:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594357:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594362:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594366:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763255.594376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763255.594379:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763255.594399:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067622912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763255.594410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763255.594415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763255.594418:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594424:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594429:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763255.594435:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763255.594442:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763255.594443:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa4700 x1876702067622400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763255.594464:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067623040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763255.594480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763255.594481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067622400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763255.594489:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763255.594497:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763255.594514:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 723 00000100:00080000:3.0:1789763255.594524:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763255.594526:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763255.594533:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763255.594541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789763255.594543:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763255.594555:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067622400:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763255.594568:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763255.594577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067623168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763255.594607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.594628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa4e00 x1876702067622528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.594646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067622528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.594657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.594662:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.594681:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763255.594713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763255.594737:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.594751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa5f80 x1876702067622656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.594766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067622656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.594776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.594780:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.594784:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763255.594792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763255.594859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.594874:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa6680 x1876702067622784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.594890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067622784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.594901:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.594905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.594909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763255.594916:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763255.594935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.594947:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa6a00 x1876702067622912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.594961:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067622912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.594971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.594974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.594978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067622912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763255.594984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763255.595005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.595017:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa6300 x1876702067623040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.595032:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067623040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.595042:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.595045:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.595049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763255.595056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763255.595076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763255.595088:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763255/real 1789763255] req@ffff936010aa4000 x1876702067623168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763255.595103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067623168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763255.595112:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763255.595116:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763255.595119:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763256.361961:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763257.386087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763258.410022:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763259.434123:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763260.458067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763260.713906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.713920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763260.713927:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.713943:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.713971:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.713983:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763260.714057:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067623296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763260.714108:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.714113:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714117:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714121:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714135:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763260.714151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067623424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763260.714181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.714211:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714215:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714221:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763260.714242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067623552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763260.714267:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763260.714273:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067623296:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763260.714275:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714285:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714295:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714301:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763260.714316:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067623680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763260.714335:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.714339:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714344:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714350:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714357:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763260.714371:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067623808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714386:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763260.714389:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.714393:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714398:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714403:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763260.714413:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714414:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763260.714434:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067623936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763260.714447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763260.714451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763260.714453:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763260.714457:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 728 00000100:00080000:3.0:1789763260.714461:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763260.714466:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763260.714473:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763260.714491:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067624064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763260.714499:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa6d80 x1876702067623296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.714586:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067623296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.714605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.714612:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.714619:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067623296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763260.714630:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763260.714661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.714678:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa5c00 x1876702067623424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.714694:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067623424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.714704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.714708:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.714712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763260.714746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763260.714770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.714783:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa7480 x1876702067623552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.714797:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067623552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.714856:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.714861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.714865:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763260.714873:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763260.714900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.714914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa5880 x1876702067623680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.714929:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067623680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.714939:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.714943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.714947:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763260.714953:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763260.714974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.714986:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa5500 x1876702067623808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.715000:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067623808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.715010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.715013:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.715018:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067623808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763260.715024:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763260.715044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.715057:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa7100 x1876702067623936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.715069:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067623936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.715079:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.715082:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.715085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067623936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763260.715092:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763260.715112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763260.715123:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763260/real 1789763260] req@ffff936010aa7b80 x1876702067624064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763260.715137:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067624064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763260.715146:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763260.715150:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763260.715153:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763261.482035:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763262.506020:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763263.530109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763264.554096:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763265.578116:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763265.833936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.833955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763265.833964:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.833984:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834006:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834021:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763265.834099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067624192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763265.834274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.834281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763265.834284:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.834294:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00100000:1.0:1789763265.834297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067624192:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763265.834302:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834313:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763265.834328:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763265.834353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067624320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763265.834371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.834376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763265.834379:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.834385:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834390:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834395:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763265.834403:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763265.834419:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067624448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763265.834431:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.834437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763265.834439:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.834444:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834448:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834453:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763265.834459:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763265.834476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067624576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763265.834494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763265.834495:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624320:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763265.834505:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763265.834511:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.834517:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834523:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834528:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763265.834535:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763265.834570:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067624704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763265.834573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624448:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763265.834604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.834614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763265.834620:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763265.834627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624576:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763265.834638:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834652:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834665:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763265.834673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624704:192.168.203.137@tcp:8:kworker.0 00000008:00100000:3.0:1789763265.834683:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763265.834717:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067624832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763265.834743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763265.834750:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763265.834753:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763265.834760:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 733 00000100:00080000:3.0:1789763265.834770:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763265.834777:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763265.834786:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763265.834794:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763265.834874:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067624960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763265.834932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763266.602017:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763267.626023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763268.650030:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763268.906179:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763268.906255:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763268.906284:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763268.906345:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763268.906363:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763268.906383:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763268.906404:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763268.906427:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763268.906435:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa5180 x1876702067624576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763268.906452:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763268.906515:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067624576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906536:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763268.906569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa7b80 x1876702067624704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906588:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067624704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906603:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906608:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763268.906615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa7800 x1876702067624320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906629:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067624320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763268.906658:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa4380 x1876702067624448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906671:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067624448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906680:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906684:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067624448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763268.906694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa4a80 x1876702067624192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067624192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906733:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067624192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763268.906789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa5500 x1876702067624960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067624960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763268.906924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763265/real 1789763268] req@ffff936010aa7100 x1876702067624832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763268.906948:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067624832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763268.906964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763268.906969:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763268.906975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067624832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763269.674019:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763270.697992:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763270.953983:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.953997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763270.954003:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954017:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954032:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954044:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763270.954101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067625088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763270.954172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763270.954181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954188:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954194:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954199:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763270.954213:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763270.954233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067625216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763270.954248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763270.954256:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954262:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954266:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763270.954276:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067625088:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763270.954276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763270.954293:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763270.954321:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067625344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763270.954343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763270.954355:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954362:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954371:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954380:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763270.954390:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763270.954396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763270.954423:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067625472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763270.954445:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789763270.954476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061dd500 x1876702067625088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763270.954484:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954504:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954518:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954531:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763270.954542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067625088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763270.954546:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763270.954579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763270.954583:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067625600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763270.954595:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.954616:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067625088:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763270.954619:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763270.954642:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763270.954649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625216:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763270.954657:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954670:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954679:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763270.954690:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763270.954718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763270.954721:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067625728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763270.954746:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763270.954758:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763270.954759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061df480 x1876702067625216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763270.954764:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763270.954784:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 738 00000100:00080000:2.0:1789763270.954841:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763270.954851:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763270.954863:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763270.954887:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067625856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763270.954940:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067625216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.954961:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.954970:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.954979:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763270.955024:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763270.955073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763270.955100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061df100 x1876702067625344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763270.955160:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067625344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.955176:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.955182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.955188:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763270.955201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763270.955246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763270.955276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061dd880 x1876702067625472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763270.955302:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067625472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.955318:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.955325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.955331:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763270.955344:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763270.955388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763270.955416:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061ddc00 x1876702067625600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763270.955443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067625600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.955474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.955481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.955487:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067625600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763270.955500:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067625728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763270.955542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763270.955567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061de300 x1876702067625728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763270.955591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067625728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.955609:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.955616:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.955622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067625728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763270.955636:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067625856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763270.955679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763270.955703:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763270/real 1789763270] req@ffff9360061ddf80 x1876702067625856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763270.955739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067625856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763270.955756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763270.955763:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763270.955771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067625856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763271.722053:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763272.746066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763273.770034:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763274.794069:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763275.818152:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763276.074040:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074054:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763276.074061:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074110:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074127:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074139:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763276.074204:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067625984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763276.074255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763276.074263:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074269:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074273:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074278:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763276.074293:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763276.074312:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067626112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763276.074327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763276.074339:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067625984:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763276.074342:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074359:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074364:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074369:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763276.074374:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763276.074393:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067626240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763276.074405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763276.074412:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074418:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074424:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074429:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763276.074436:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763276.074452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067626368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763276.074459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763276.074482:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074490:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763276.074495:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074505:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074513:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074522:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763276.074549:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763276.074574:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067626496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763276.074579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa5880 x1876702067625984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763276.074618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763276.074628:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074637:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00100000:1.0:1789763276.074638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067625984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763276.074646:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074660:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763276.074667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763276.074670:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763276.074679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.074687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067625984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763276.074695:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067626624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763276.074716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763276.074720:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763276.074722:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763276.074727:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 743 00000100:00080000:3.0:1789763276.074731:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763276.074734:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763276.074740:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763276.074747:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763276.074757:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067626752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763276.074794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.074876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa7480 x1876702067626112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.074903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067626112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.074920:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.074928:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.074936:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763276.074951:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763276.074996:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.075021:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa5c00 x1876702067626240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.075047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067626240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.075065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.075100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.075123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763276.075132:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763276.075165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.075200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa6d80 x1876702067626368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.075215:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067626368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.075226:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.075230:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.075234:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763276.075241:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763276.075263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.075277:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa4000 x1876702067626496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.075290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067626496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.075299:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.075303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.075307:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067626496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763276.075314:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067626624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763276.075334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.075346:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa6300 x1876702067626624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.075359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067626624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.075369:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.075372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.075376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067626624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763276.075404:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067626752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763276.075425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763276.075438:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763276/real 1789763276] req@ffff936010aa6a00 x1876702067626752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763276.075451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067626752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763276.075462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763276.075466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763276.075470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067626752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763276.842060:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763277.865989:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763278.890094:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763279.914101:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763280.938121:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763281.193967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.193980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763281.193986:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194000:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763281.194112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067626880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763281.194165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.194173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763281.194177:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194186:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194195:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194205:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763281.194220:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763281.194246:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067627008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763281.194249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067626880:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763281.194286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.194291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763281.194294:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194299:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194304:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194309:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763281.194317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763281.194332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067627136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763281.194345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.194350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763281.194352:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194358:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 749 00000800:00000100:1.0:1789763281.194367:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763281.194369:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194378:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763281.194385:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763281.194401:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067627264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763281.194414:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.194418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763281.194421:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194425:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194431:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194437:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763281.194445:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763281.194448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020dac380 x1876702067626880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763281.194465:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067627392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763281.194490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763281.194496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763281.194502:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763281.194506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067626880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763281.194517:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194535:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763281.194541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763281.194550:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763281.194556:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.194566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067626880:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763281.194569:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763281.194598:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067627520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763281.194616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763281.194617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627008:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763281.194627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763281.194637:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763281.194649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 749 00000100:00080000:0.0:1789763281.194658:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763281.194667:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763281.194678:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763281.194689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763281.194709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067627648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763281.194720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020dae680 x1876702067627008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.194769:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067627008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.194787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.194792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.194860:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763281.194872:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763281.194903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763281.194919:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020daed80 x1876702067627136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.194935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067627136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.194945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.194950:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.194954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763281.194960:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763281.194982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763281.194994:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020daf480 x1876702067627264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.195008:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067627264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.195018:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.195021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.195026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763281.195065:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763281.195094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763281.195109:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020dafb80 x1876702067627392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.195124:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067627392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.195134:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.195138:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.195142:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763281.195165:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067627520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763281.195186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763281.195199:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020dac000 x1876702067627520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.195214:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067627520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.195223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.195227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.195230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067627520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763281.195237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067627648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763281.195257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763281.195270:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763281/real 1789763281] req@ffff936020daca80 x1876702067627648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763281.195284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067627648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763281.195294:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763281.195298:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763281.195301:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067627648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763281.961975:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763282.986045:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763284.010043:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763285.034043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763286.058006:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763286.313926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.313944:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763286.313947:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.313973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.313983:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.313990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763286.314049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067627776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.314073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763286.314074:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314077:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.314080:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.314083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763286.314091:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763286.314099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067627904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.314109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763286.314110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314113:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.314115:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.314119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067627776:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763286.314123:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763286.314134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067628032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.314144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763286.314145:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314148:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.314150:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.314152:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763286.314156:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763286.314163:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067628160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.314172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763286.314174:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314176:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.314179:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.314181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763286.314186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763286.314187:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763286.314198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067628288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763286.314207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763286.314208:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314210:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:3.0:1789763286.314212:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763286.314215:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763286.314218:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763286.314227:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa6680 x1876702067627776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763286.314231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067628416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763286.314252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763286.314255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067627776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763286.314257:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763286.314262:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763286.314287:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 754 00000100:00080000:1.0:1789763286.314288:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763286.314294:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763286.314295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314302:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067627776:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763286.314305:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314312:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627904:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763286.314313:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763286.314332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067628544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763286.314356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa5f80 x1876702067627904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314383:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067627904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314389:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314391:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314393:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067627904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763286.314414:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763286.314425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314432:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa4e00 x1876702067628032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314439:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067628032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314444:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314445:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763286.314457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763286.314468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa4700 x1876702067628160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067628160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314487:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314491:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763286.314494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763286.314504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa6a00 x1876702067628288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067628288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763286.314527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067628416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763286.314537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314543:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa6300 x1876702067628416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314550:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067628416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314554:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314556:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314559:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067628416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763286.314563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067628544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763286.314573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763286.314587:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763286/real 1789763286] req@ffff936010aa4000 x1876702067628544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763286.314594:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067628544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763286.314599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763286.314601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763286.314603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067628544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763287.082074:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763288.105923:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763289.130047:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763290.153955:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763291.178057:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763291.433945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763291.433984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763291.433993:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763291.434009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434026:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434037:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763291.434097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067628672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763291.434147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763291.434150:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763291.434155:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434159:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434164:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763291.434177:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763291.434194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067628800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763291.434220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763291.434222:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763291.434227:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434238:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763291.434245:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763291.434259:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067628928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763291.434280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763291.434286:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763291.434288:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067628672:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763291.434296:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434305:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434310:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763291.434317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763291.434333:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067629056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763291.434351:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763291.434354:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763291.434359:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434365:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434369:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763291.434374:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763291.434387:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067629184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763291.434404:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763291.434407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763291.434414:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763291.434422:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434430:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763291.434436:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763291.434443:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763291.434458:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067629312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763291.434475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763291.434476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa6d80 x1876702067628672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763291.434496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763291.434508:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763291.434521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067628672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763291.434523:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 759 00000100:00080000:3.0:1789763291.434541:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763291.434548:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763291.434558:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763291.434562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.434576:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067628672:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763291.434579:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763291.434589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763291.434614:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067629440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763291.434631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.434656:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa5c00 x1876702067628800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.434674:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067628800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.434684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.434689:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.434693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763291.434727:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763291.434749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.434762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa7480 x1876702067628928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.434775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067628928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.434785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.434789:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.434793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067628928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763291.434847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763291.434874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.434889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa5880 x1876702067629056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.434904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067629056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.434915:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.434919:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.434923:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763291.434929:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763291.434949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.434994:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa7100 x1876702067629184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.435010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067629184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.435020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.435024:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.435028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763291.435035:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067629312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763291.435059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.435072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa5500 x1876702067629312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.435086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067629312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.435096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.435100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.435104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067629312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763291.435110:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067629440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763291.435130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763291.435141:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763291/real 1789763291] req@ffff936010aa4a80 x1876702067629440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763291.435154:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067629440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763291.435163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763291.435167:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763291.435171:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067629440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763292.202099:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763293.226076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763294.249956:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763295.273943:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763296.297936:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763296.553873:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.553884:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763296.553888:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.553899:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.553910:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763296.553928:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763296.553966:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067629568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763296.553994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.553998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763296.554000:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.554005:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.554009:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763296.554015:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763296.554025:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763296.554033:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067629568:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763296.554040:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067629696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763296.554055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.554058:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763296.554060:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.554064:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.554068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763296.554073:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763296.554078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763296.554091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067629824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763296.554095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763296.554108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.554112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763296.554114:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.554119:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.554124:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763296.554128:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554132:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa4380 x1876702067629568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763296.554136:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763296.554151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067629568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763296.554157:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067629952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763296.554164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763296.554172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763296.554175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763296.554177:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067629568:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763296.554183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763296.554187:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.554192:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.554196:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763296.554200:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629696:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763296.554207:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763296.554223:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067630080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763296.554233:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763296.554241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.554246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763296.554247:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa7800 x1876702067629696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763296.554249:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763296.554260:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00100000:1.0:1789763296.554265:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067629696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763296.554267:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763296.554277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763296.554277:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763296.554285:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789763296.554287:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763296.554289:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763296.554297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763296.554305:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067630208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763296.554314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554323:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa7b80 x1876702067629824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763296.554345:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067629824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763296.554346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763296.554355:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763296.554359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763296.554360:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763296.554364:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763296.554370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629824:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763296.554370:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 764 00000100:00080000:3.0:1789763296.554379:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763296.554380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629952:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763296.554389:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763296.554396:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763296.554400:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa5180 x1876702067629952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763296.554417:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067630336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763296.554422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067629952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763296.554445:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763296.554449:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763296.554452:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067629952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763296.554463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763296.554477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554483:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa4380 x1876702067630080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763296.554491:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067630080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763296.554496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763296.554498:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763296.554499:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763296.554503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067630208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763296.554514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa4a80 x1876702067630208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763296.554527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067630208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763296.554532:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763296.554534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763296.554536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067630208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763296.554539:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067630336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763296.554555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763296.554562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763296/real 1789763296] req@ffff936010aa5500 x1876702067630336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763296.554568:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067630336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763296.554573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763296.554575:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763296.554577:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067630336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763297.321952:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763298.345940:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763299.369949:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763300.393992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763301.417936:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763301.673913:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763301.673927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763301.673932:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.673942:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.673953:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.673960:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763301.673999:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067630464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763301.674028:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763301.674032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763301.674034:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.674042:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.674047:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763301.674058:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763301.674070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067630464:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763301.674076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067630592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763301.674091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763301.674096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763301.674098:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674103:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.674107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.674113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763301.674118:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763301.674131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067630720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763301.674140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763301.674147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763301.674152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763301.674154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674160:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.674164:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.674169:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763301.674175:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763301.674187:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067630848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763301.674197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763301.674200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763301.674210:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674215:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.674219:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.674223:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674229:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061dc380 x1876702067630464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763301.674231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763301.674251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067630976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763301.674253:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067630464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674285:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763301.674286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763301.674291:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763301.674295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763301.674297:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067630464:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763301.674300:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674308:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00080000:2.0:1789763301.674312:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763301.674317:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674323:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630592:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789763301.674323:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763301.674344:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067631104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763301.674351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763301.674364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763301.674367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061dc700 x1876702067630592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763301.674371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763301.674376:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763301.674384:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 769 00000100:00100000:1.0:1789763301.674385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067630592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763301.674392:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763301.674396:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763301.674402:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763301.674404:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630592:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789763301.674414:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763301.674417:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763301.674435:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067631232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763301.674441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061ddf80 x1876702067630720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763301.674475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067630720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674492:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763301.674495:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674497:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763301.674501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763301.674519:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061de300 x1876702067630848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763301.674534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067630848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763301.674542:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674544:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763301.674547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763301.674557:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061ddc00 x1876702067630976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763301.674570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067630976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763301.674576:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674579:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067630976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763301.674591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067631104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763301.674603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061dd880 x1876702067631104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763301.674617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067631104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763301.674623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067631104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763301.674629:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067631232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763301.674639:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763301.674645:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763301/real 1789763301] req@ffff9360061df100 x1876702067631232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763301.674652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067631232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763301.674656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763301.674658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763301.674660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067631232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763302.441959:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763303.465956:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763304.489979:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763305.513948:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763306.537954:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763306.793913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763306.793927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763306.793931:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763306.793942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.793954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.793961:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763306.793998:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067631360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763306.794025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763306.794029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763306.794031:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763306.794036:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794040:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.794044:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763306.794054:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763306.794067:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067631360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763306.794071:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067631488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763306.794085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763306.794089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763306.794091:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763306.794097:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.794105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763306.794109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763306.794122:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067631616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763306.794128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763306.794138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763306.794142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763306.794144:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763306.794149:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794152:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.794157:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763306.794162:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763306.794176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa7100 x1876702067631360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763306.794176:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067631744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763306.794202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763306.794206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763306.794209:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763306.794215:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067631360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763306.794215:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794222:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.794230:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763306.794233:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763306.794236:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763306.794239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794245:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067631360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763306.794254:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067631872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763306.794265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763306.794269:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631488:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763306.794271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763306.794277:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763306.794284:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794289:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763306.794294:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763306.794298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763306.794303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763306.794312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa5880 x1876702067631488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763306.794319:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067632000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763306.794326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067631488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763306.794340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763306.794342:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763306.794347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763306.794352:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763306.794353:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763306.794362:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631488:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763306.794363:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 774 00000100:00080000:3.0:1789763306.794369:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763306.794377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631616:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763306.794378:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763306.794386:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763306.794400:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067632128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763306.794412:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763306.794425:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa7480 x1876702067631616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763306.794435:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067631616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763306.794441:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763306.794444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794446:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763306.794450:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763306.794462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763306.794468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa5c00 x1876702067631744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763306.794476:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067631744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763306.794481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763306.794483:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794485:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763306.794488:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763306.794498:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763306.794504:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa6d80 x1876702067631872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763306.794517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067631872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763306.794522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763306.794524:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794525:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067631872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763306.794537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763306.794548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763306.794555:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa4000 x1876702067632000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763306.794561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067632000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763306.794566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763306.794568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763306.794573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763306.794583:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763306.794589:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763306/real 1789763306] req@ffff936010aa6300 x1876702067632128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763306.794596:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067632128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763306.794611:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763306.794614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763306.794618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763307.561915:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763308.585954:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763309.609959:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763310.633971:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763311.658029:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763311.913943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.913957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763311.913962:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.913977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.913991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914000:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763311.914061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067632256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763311.914104:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914109:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914113:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914117:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763311.914139:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763311.914156:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067632384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763311.914182:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067632256:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763311.914183:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914194:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914204:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914209:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763311.914215:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763311.914230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067632512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763311.914249:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914254:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914259:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914264:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763311.914269:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763311.914281:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067632640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914292:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1789763311.914304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763311.914306:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914316:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914321:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914326:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763311.914332:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763311.914345:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067632768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763311.914364:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914368:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914372:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914376:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763311.914382:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763311.914389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa6a00 x1876702067632256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763311.914403:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067632896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763311.914422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763311.914428:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763311.914432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067632256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763311.914432:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763311.914448:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 779 00000100:00080000:3.0:1789763311.914457:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763311.914467:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763311.914472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763311.914473:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763311.914489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763311.914496:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067633024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763311.914504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067632256:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763311.914556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763311.914599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.914615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa4700 x1876702067632384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.914631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067632384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.914655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.914660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.914664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763311.914672:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763311.914695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.914709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa4e00 x1876702067632512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.914723:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067632512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.914739:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.914745:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.914751:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763311.914762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763311.914887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.914908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa5f80 x1876702067632640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.914935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067632640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.914945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.914949:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.914954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763311.914962:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763311.914990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.915003:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa6680 x1876702067632768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.915017:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067632768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.915026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.915039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.915043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067632768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763311.915049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763311.915070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.915081:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa6300 x1876702067632896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.915096:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067632896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.915105:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.915108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.915112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067632896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763311.915118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763311.915136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763311.915148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763311/real 1789763311] req@ffff936010aa4000 x1876702067633024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763311.915162:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067633024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763311.915172:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763311.915177:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763311.915184:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763312.682067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763313.706039:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763314.730128:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763315.754069:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763316.777980:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763317.033888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.033900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763317.033903:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.033912:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.033922:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.033928:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763317.033961:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067633152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763317.033985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.033988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763317.033989:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.033992:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.033995:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.033997:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034006:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763317.034016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067633280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763317.034025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763317.034028:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067633152:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763317.034029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763317.034032:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.034039:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.034041:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.034043:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034047:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763317.034055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067633408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763317.034063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.034065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763317.034067:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.034070:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.034073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.034076:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763317.034086:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067633536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763317.034093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763317.034095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.034098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763317.034100:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.034104:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.034107:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.034110:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034112:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763317.034119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067633664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763317.034125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.034127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763317.034129:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.034133:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00100000:1.0:1789763317.034134:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa6d80 x1876702067633152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763317.034136:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.034141:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034148:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763317.034162:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067633152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763317.034163:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067633792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763317.034193:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034200:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763317.034200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763317.034206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763317.034208:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067633152:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763317.034209:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763317.034218:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 784 00000100:00080000:3.0:1789763317.034221:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763317.034223:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763317.034226:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763317.034232:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067633920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763317.034236:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763317.034261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034271:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa5c00 x1876702067633280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034280:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067633280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763317.034294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763317.034321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa7480 x1876702067633408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034340:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067633408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034349:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034352:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763317.034356:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763317.034370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa5880 x1876702067633536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034388:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067633536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034395:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763317.034405:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763317.034418:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034452:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa7100 x1876702067633664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034459:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067633664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034464:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067633664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763317.034471:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763317.034483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa5500 x1876702067633792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034512:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067633792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034519:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763317.034543:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763317.034558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763317.034567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763317/real 1789763317] req@ffff936010aa4a80 x1876702067633920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763317.034577:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067633920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763317.034583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763317.034585:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763317.034588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067633920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763317.802041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763318.826036:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763319.849985:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763320.873937:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763321.898041:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763322.153892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.153903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763322.153906:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.153916:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.153925:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.153931:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763322.153965:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067634048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.153987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.153990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763322.153992:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.153994:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.153997:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.153999:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763322.154007:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763322.154015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067634176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.154021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.154023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763322.154025:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.154027:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.154029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.154031:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763322.154036:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763322.154043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067634304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.154049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.154051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763322.154052:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.154055:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.154057:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.154062:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154063:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067634048:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763322.154066:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763322.154075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067634432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.154081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.154083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763322.154084:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.154086:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.154088:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.154090:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763322.154093:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763322.154103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067634560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.154108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.154110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763322.154111:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.154114:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.154117:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.154119:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763322.154122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763322.154128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763322.154132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067634688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763322.154140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763322.154142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763322.154144:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763322.154147:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 789 00000100:00080000:0.0:1789763322.154149:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763322.154151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763322.154154:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763322.154161:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067634816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763322.154172:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020dad880 x1876702067634048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154189:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067634048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154198:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154201:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154204:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067634048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763322.154222:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763322.154245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154259:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020dac700 x1876702067634176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067634176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154282:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154286:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154289:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763322.154294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763322.154310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020daf100 x1876702067634304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154325:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067634304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154331:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154335:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763322.154355:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763322.154366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154373:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020dad500 x1876702067634432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154381:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067634432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154386:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154387:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154389:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763322.154392:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763322.154402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020dadf80 x1876702067634560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154415:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067634560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154420:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154422:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067634560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763322.154427:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067634688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763322.154438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020daf800 x1876702067634688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067634688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154463:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067634688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763322.154468:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067634816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763322.154478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763322.154484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763322/real 1789763322] req@ffff936020dae300 x1876702067634816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763322.154499:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067634816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763322.154505:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763322.154508:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763322.154510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067634816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763322.921915:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763323.946063:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763324.970047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763325.994021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763327.018127:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763327.274013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763327.274032:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274044:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274058:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274067:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763327.274130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067634944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763327.274170:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274176:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274198:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274203:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763327.274215:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763327.274231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067635072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274244:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763327.274250:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274255:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274259:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274267:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763327.274271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067634944:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763327.274274:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763327.274307:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067635200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274321:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763327.274329:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274333:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274337:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274342:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763327.274348:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763327.274363:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067635328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763327.274398:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274402:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274406:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274410:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763327.274415:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763327.274425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763327.274432:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067635456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274447:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763327.274449:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763327.274453:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274459:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274463:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763327.274468:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763327.274497:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067635584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763327.274510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763327.274515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763327.274518:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763327.274527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa4380 x1876702067634944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763327.274529:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 795 00000100:00080000:3.0:1789763327.274539:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763327.274568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763327.274579:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763327.274592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067634944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763327.274598:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067635712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763327.274655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.274661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.274685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067634944:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763327.274718:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763327.274749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.274785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa5180 x1876702067635072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.274800:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067635072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.274810:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.274886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.274892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763327.274928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763327.274977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.274991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa7b80 x1876702067635200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.275006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067635200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.275016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.275019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.275023:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763327.275030:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763327.275051:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.275062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa7800 x1876702067635328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.275076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067635328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.275085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.275088:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.275091:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763327.275097:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763327.275134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.275145:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa4a80 x1876702067635456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.275156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067635456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.275164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.275167:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.275170:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763327.275176:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067635584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763327.275213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.275224:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa5500 x1876702067635584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.275263:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067635584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.275280:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.275286:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.275292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067635584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763327.275305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067635712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763327.275361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763327.275383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763327/real 1789763327] req@ffff936010aa7100 x1876702067635712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763327.275416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067635712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763327.275426:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763327.275429:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763327.275432:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067635712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763328.042025:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763329.065921:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763330.090014:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763331.114076:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763332.138002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763332.393925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.393939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763332.393945:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.393958:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.393973:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.393982:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763332.394031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067635840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763332.394077:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.394086:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394102:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763332.394118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067635968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763332.394138:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394143:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.394147:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394156:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763332.394166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067636096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763332.394185:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394191:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.394195:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394201:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763332.394223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067635840:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763332.394226:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067636224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394247:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763332.394253:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394259:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.394263:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394267:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394275:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763332.394292:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067636352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394304:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763332.394311:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394314:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00080000:3.0:1789763332.394320:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394325:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394330:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763332.394344:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067636480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763332.394356:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763332.394362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763332.394364:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763332.394371:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 800 00000100:00100000:1.0:1789763332.394378:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635968:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763332.394380:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763332.394395:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763332.394405:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763332.394423:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067636608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763332.394436:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763332.394484:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763332.394511:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763332.394537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067636480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763332.394565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067636608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763333.161976:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763334.186130:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763335.210050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763335.466190:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763335.466241:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763335.466260:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466308:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466317:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466324:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466333:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466340:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763335.466347:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763335.466374:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa6d80 x1876702067636224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466408:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067636224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466436:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763335.466447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa4000 x1876702067636352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466463:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067636352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466479:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763335.466494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa6300 x1876702067636480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466509:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067636480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067636480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763335.466533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa6680 x1876702067636608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466552:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067636608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466560:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466564:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067636608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763335.466575:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa5880 x1876702067635840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466586:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067635840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067635840:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763335.466608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa7480 x1876702067635968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067635968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067635968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763335.466654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763332/real 1789763335] req@ffff936010aa5c00 x1876702067636096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763335.466668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067636096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763335.466675:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763335.466679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763335.466682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636096:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763336.234071:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763337.258086:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763337.513898:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.513909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763337.513912:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.513925:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.513935:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.513942:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763337.513981:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067636736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763337.514014:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514018:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514022:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.514024:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514033:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763337.514041:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067636864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514048:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763337.514052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514055:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514057:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.514060:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514064:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763337.514072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067636992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763337.514083:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514086:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514088:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763337.514094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067636736:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763337.514094:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514102:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763337.514110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067637120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763337.514121:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514124:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.514129:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514131:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763337.514141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067637248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763337.514151:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514155:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514158:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.514161:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514164:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763337.514171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763337.514173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067637376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763337.514181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763337.514184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763337.514186:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763337.514189:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 805 00000100:00080000:3.0:1789763337.514192:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763337.514195:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763337.514198:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763337.514205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067637504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763337.514226:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa5f80 x1876702067636736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514244:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067636736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514264:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067636736:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763337.514270:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763337.514290:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514301:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa4e00 x1876702067636864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514315:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067636864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514324:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514331:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763337.514339:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763337.514362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa4700 x1876702067636992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514381:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067636992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514391:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514394:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067636992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763337.514431:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763337.514474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa6a00 x1876702067637120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514499:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067637120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514505:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514507:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763337.514514:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763337.514530:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514537:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa5c00 x1876702067637248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514550:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067637248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763337.514572:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067637376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763337.514594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514607:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa7480 x1876702067637376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514643:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067637376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067637376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763337.514666:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067637504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763337.514684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763337.514693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763337/real 1789763337] req@ffff936010aa5880 x1876702067637504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763337.514701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067637504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763337.514707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763337.514710:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763337.514714:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067637504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763338.281988:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763339.306126:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763340.330054:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763341.354046:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763342.377964:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763342.633953:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.633964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763342.633970:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.633982:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634007:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634017:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763342.634092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067637632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763342.634135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763342.634142:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634150:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763342.634184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067637760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763342.634195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763342.634204:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634211:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00100000:1.0:1789763342.634212:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067637632:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763342.634215:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634238:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763342.634259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067637888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763342.634267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763342.634273:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634279:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634281:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634285:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634288:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763342.634300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067638016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763342.634307:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634310:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763342.634312:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634316:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634319:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634322:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634328:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763342.634340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067638144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763342.634351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763342.634362:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763342.634368:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634382:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634388:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634395:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634399:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763342.634410:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067638272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763342.634417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763342.634420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763342.634422:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763342.634427:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 810 00000100:00080000:2.0:1789763342.634430:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763342.634435:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763342.634440:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763342.634450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061df480 x1876702067637632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763342.634456:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067638400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763342.634558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067637632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.634573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.634602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.634608:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067637632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763342.634642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763342.634676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.634692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061dd500 x1876702067637760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.634708:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067637760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.634718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.634722:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.634726:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763342.634733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763342.634754:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.634766:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061dca80 x1876702067637888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.634779:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067637888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.634837:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.634843:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.634847:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067637888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763342.634856:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763342.634897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.634914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061df800 x1876702067638016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.634930:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067638016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.634940:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.634944:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.634948:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763342.634955:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763342.634979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.635016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061dfb80 x1876702067638144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.635038:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067638144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.635053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.635060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.635067:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763342.635080:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067638272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763342.635135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.635152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061de680 x1876702067638272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.635167:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067638272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.635177:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.635181:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.635185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067638272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763342.635193:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067638400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763342.635214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763342.635226:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763342/real 1789763342] req@ffff9360061dc000 x1876702067638400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763342.635259:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067638400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763342.635277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763342.635281:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763342.635284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067638400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763343.402047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763344.426044:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763345.450031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763346.474062:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763347.497965:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763347.753972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763347.753984:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763347.753989:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754000:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754013:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763347.754022:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763347.754068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067638528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763347.754108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763347.754115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763347.754119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754126:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763347.754148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754162:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763347.754185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067638656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763347.754207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763347.754216:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067638528:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763347.754217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763347.754225:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763347.754245:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754252:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763347.754271:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067638784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763347.754284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763347.754288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763347.754290:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754295:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754300:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763347.754306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763347.754311:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754320:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763347.754335:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067638912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763347.754346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763347.754350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763347.754352:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754357:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754362:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763347.754364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa6680 x1876702067638528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763347.754370:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754398:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763347.754410:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067638528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763347.754417:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067639040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763347.754424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.754434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.754459:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067638528:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763347.754472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763347.754481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763347.754485:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754492:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754499:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763347.754505:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638656:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763347.754509:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754520:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763347.754539:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067639168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763347.754574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763347.754594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763347.754600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa6300 x1876702067638656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763347.754602:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763347.754607:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763347.754620:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 815 00000100:00080000:3.0:1789763347.754630:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763347.754636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067638656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763347.754639:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763347.754650:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763347.754656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.754663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.754677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763347.754683:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067639296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763347.754693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763347.754731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763347.754745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa4000 x1876702067638784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763347.754758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067638784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763347.754766:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.754769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.754772:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763347.754778:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763347.754829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763347.754843:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa6d80 x1876702067638912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763347.754878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067638912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763347.754893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.754896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.754899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067638912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763347.754903:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763347.754921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763347.754929:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa7100 x1876702067639040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763347.754938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067639040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763347.754955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.754957:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.754960:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763347.754975:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067639168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763347.754989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763347.754997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa5500 x1876702067639168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763347.755006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067639168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763347.755012:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.755014:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.755017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067639168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763347.755021:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067639296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763347.755033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763347.755040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763347/real 1789763347] req@ffff936010aa4a80 x1876702067639296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763347.755049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067639296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763347.755055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763347.755057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763347.755060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067639296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763348.521938:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763349.546058:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763350.569946:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763351.594004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763352.618056:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763352.873928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.873942:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763352.873948:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.873963:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.873977:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.873986:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763352.874036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067639424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.874083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763352.874085:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874090:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874094:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.874100:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874112:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763352.874126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067639552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.874146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763352.874148:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874153:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874157:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763352.874167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067639424:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763352.874167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874179:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763352.874194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067639680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.874212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763352.874214:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874219:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874223:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.874227:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874234:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763352.874248:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067639808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.874265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763352.874267:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874271:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874276:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.874284:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763352.874289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763352.874309:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067639936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763352.874326:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763352.874329:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874334:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874337:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.874342:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874347:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763352.874360:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067640064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763352.874371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763352.874379:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa7800 x1876702067639424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763352.874380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763352.874387:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763352.874401:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 820 00000100:00080000:3.0:1789763352.874412:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763352.874417:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763352.874424:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763352.874430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067639424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763352.874445:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067640192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763352.874460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.874475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.874486:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067639424:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763352.874502:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763352.874576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.874597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa7b80 x1876702067639552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.874615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067639552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.874627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.874631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.874635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763352.874669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763352.874695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.874708:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa5180 x1876702067639680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.874723:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067639680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.874733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.874737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.874741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763352.874748:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763352.874768:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.874780:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa4380 x1876702067639808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.874795:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067639808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.874846:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.874851:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.874855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763352.874863:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763352.874889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.874903:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa4a80 x1876702067639936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.874919:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067639936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.874929:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.874933:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.874936:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067639936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763352.874943:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763352.874963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.874975:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa5500 x1876702067640064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.874989:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067640064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.874998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.875001:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.875005:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763352.875011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763352.875031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763352.875042:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763352/real 1789763352] req@ffff936010aa7100 x1876702067640192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763352.875055:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067640192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763352.875073:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763352.875077:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763352.875081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763353.642070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763354.666122:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763355.690124:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763356.714136:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763357.737995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763357.993966:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.993980:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763357.993986:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763357.994002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994016:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763357.994025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763357.994074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067640320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763357.994122:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994127:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763357.994130:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763357.994135:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763357.994144:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763357.994156:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763357.994170:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067640448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763357.994184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763357.994192:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763357.994197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994209:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763357.994213:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763357.994219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763357.994226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067640320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763357.994239:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067640576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763357.994263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763357.994270:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763357.994276:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994280:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763357.994285:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763357.994291:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763357.994305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067640704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763357.994316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763357.994325:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763357.994330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763357.994334:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994342:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763357.994348:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763357.994355:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763357.994370:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067640832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763357.994381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994385:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763357.994388:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763357.994393:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994401:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763357.994409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061dce00 x1876702067640320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763357.994412:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763357.994426:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763357.994452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067640960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763357.994460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067640320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763357.994497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763357.994510:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763357.994513:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763357.994537:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763357.994538:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 825 00000100:00080000:2.0:1789763357.994553:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763357.994557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763357.994565:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763357.994573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067640320:192.168.203.137@tcp:250:kworker.0 00000008:00100000:2.0:1789763357.994578:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763357.994591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763357.994602:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067641088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763357.994643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.994666:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061dd180 x1876702067640448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.994689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067640448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.994707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.994713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.994720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763357.994763:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763357.994864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.994902:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061dc000 x1876702067640576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.994919:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067640576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.994931:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.994935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.994939:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763357.994947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763357.994974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.994988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061de680 x1876702067640704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.995003:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067640704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.995013:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.995017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.995022:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763357.995029:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763357.995049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.995061:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061dfb80 x1876702067640832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.995075:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067640832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.995085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.995088:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.995096:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067640832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763357.995102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763357.995128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.995148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061df800 x1876702067640960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.995173:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067640960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.995189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.995196:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.995203:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067640960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763357.995221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763357.995251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763357.995267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763357/real 1789763357] req@ffff9360061dca80 x1876702067641088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763357.995291:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067641088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763357.995307:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763357.995314:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763357.995321:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763358.762030:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763359.786084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763360.809926:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763361.834065:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763362.858056:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763363.113980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.113997:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763363.114004:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763363.114019:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114037:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114049:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763363.114131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067641216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763363.114172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763363.114181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763363.114192:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114197:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114202:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763363.114218:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763363.114238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067641344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763363.114254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763363.114264:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763363.114271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067641216:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763363.114273:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114284:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114293:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763363.114300:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763363.114315:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067641472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763363.114328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763363.114335:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763363.114341:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114359:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114368:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763363.114377:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763363.114409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763363.114410:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067641600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763363.114461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114469:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763363.114474:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763363.114482:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114489:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114496:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763363.114505:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763363.114520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa6d80 x1876702067641216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763363.114533:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067641728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763363.114580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067641216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763363.114590:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763363.114618:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763363.114623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763363.114629:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114639:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763363.114641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763363.114652:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763363.114657:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067641216:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763363.114664:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763363.114689:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067641856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763363.114703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763363.114707:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763363.114710:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763363.114717:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641344:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763363.114721:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 830 00000100:00080000:3.0:1789763363.114731:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763363.114738:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763363.114746:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763363.114765:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067641984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763363.114789:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.114894:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa4000 x1876702067641344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.114923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067641344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.114942:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.114949:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.114957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763363.114970:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763363.115013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.115038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa6300 x1876702067641472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.115066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067641472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.115094:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.115099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.115104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763363.115113:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763363.115146:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.115161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa6680 x1876702067641600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.115176:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067641600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.115187:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.115191:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.115196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763363.115202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763363.115222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.115234:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa5880 x1876702067641728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.115248:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067641728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.115258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.115262:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.115265:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067641728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763363.115271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763363.115291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.115302:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa7480 x1876702067641856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.115315:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067641856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.115325:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.115328:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.115333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763363.115359:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763363.115381:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763363.115393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763363/real 1789763363] req@ffff936010aa5c00 x1876702067641984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763363.115407:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067641984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763363.115417:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763363.115420:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763363.115424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067641984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763363.882051:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763364.906009:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763365.929911:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763366.954042:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763367.978038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763368.233967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.233977:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763368.233983:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.233995:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234010:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763368.234083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067642112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763368.234132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763368.234168:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234174:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234187:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763368.234224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067642240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763368.234239:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763368.234246:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234254:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234257:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234261:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234267:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763368.234284:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067642112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763368.234285:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067642368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763368.234341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763368.234347:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234356:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234361:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234365:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763368.234385:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067642496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763368.234413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763368.234420:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234425:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234430:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234434:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234449:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763368.234458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763368.234467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067642624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763368.234481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763368.234488:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234492:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234495:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234498:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234503:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763368.234521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067642752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763368.234535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020dadc00 x1876702067642112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763368.234545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763368.234555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763368.234569:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763368.234578:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 836 00000100:00080000:0.0:1789763368.234586:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763368.234602:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763368.234610:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763368.234625:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067642880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763368.234644:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067642112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.234668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.234676:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.234684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067642112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763368.234697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763368.234731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.234748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020dace00 x1876702067642240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.234764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067642240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.234774:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.234778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.234783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763368.234816:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763368.234855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.234926:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020daea00 x1876702067642368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.234942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067642368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.234952:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.234956:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.234960:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763368.234967:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763368.234992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.235018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020dad180 x1876702067642496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.235033:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067642496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.235043:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.235046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.235049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763368.235056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763368.235078:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.235090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020dae300 x1876702067642624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.235102:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067642624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.235111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.235114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.235117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067642624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763368.235123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067642752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763368.235162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.235174:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020daf800 x1876702067642752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.235188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067642752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.235206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.235210:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.235214:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067642752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763368.235220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067642880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763368.235241:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763368.235253:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763368/real 1789763368] req@ffff936020dadf80 x1876702067642880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763368.235266:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067642880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763368.235274:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763368.235278:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763368.235281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067642880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763369.001980:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763370.026009:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763371.050030:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763372.073922:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763373.098061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763373.354031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763373.354053:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763373.354068:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354087:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354098:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763373.354161:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067643008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763373.354205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763373.354213:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763373.354218:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354222:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354227:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763373.354241:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763373.354258:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067643136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763373.354275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354287:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763373.354289:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067643008:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763373.354294:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763373.354310:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354317:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354326:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763373.354337:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763373.354391:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067643264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763373.354411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763373.354420:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763373.354428:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763373.354431:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354441:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354450:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763373.354459:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763373.354493:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067643392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763373.354504:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa6a00 x1876702067643008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763373.354544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1789763373.354557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067643008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763373.354561:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763373.354583:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354595:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354607:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763373.354608:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.354620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789763373.354620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763373.354635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067643008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763373.354660:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067643520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763373.354677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643136:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763373.354693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763373.354709:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763373.354723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763373.354724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 841 00000100:00080000:3.0:1789763373.354739:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354749:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763373.354755:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa4700 x1876702067643136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763373.354767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763373.354778:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067643136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.354791:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.354860:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.354868:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763373.354882:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763373.354884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067643648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763373.354907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763373.354912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763373.354916:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763373.354924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 841 00000800:00000100:1.0:1789763373.354932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763373.354938:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763373.354954:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763373.354960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa4e00 x1876702067643264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763373.354970:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763373.354987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067643264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.355002:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763373.355010:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067643776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763373.355012:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.355027:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763373.355050:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763373.355083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763373.355104:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa5f80 x1876702067643392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763373.355119:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067643392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.355129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.355133:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.355137:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763373.355157:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763373.355179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763373.355192:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa5c00 x1876702067643520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763373.355206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067643520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.355216:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.355219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.355223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067643520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763373.355230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067643648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763373.355250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763373.355262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa7480 x1876702067643648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763373.355277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067643648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.355287:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.355290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.355294:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067643648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763373.355300:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067643776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763373.355319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763373.355330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763373/real 1789763373] req@ffff936010aa5880 x1876702067643776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763373.355343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067643776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763373.355375:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763373.355380:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763373.355384:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067643776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763374.122066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763375.145975:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763376.170096:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763377.194099:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763378.217952:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763378.473975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.473990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763378.473996:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474010:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474025:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474034:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763378.474103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067643904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763378.474149:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474154:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474159:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474164:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763378.474190:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067644032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763378.474232:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474240:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474244:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474250:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763378.474263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067644160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474276:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763378.474282:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474286:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474289:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474293:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474299:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763378.474340:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067643904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763378.474343:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067644288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763378.474410:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474416:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474422:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474426:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474433:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763378.474450:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067644416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763378.474468:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474472:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474477:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474482:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474486:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763378.474500:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067644544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763378.474510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763378.474518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1789763378.474523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763378.474524:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763378.474540:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 846 00000100:00080000:3.0:1789763378.474546:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763378.474552:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763378.474575:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763378.474595:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067644672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763378.474642:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa6680 x1876702067643904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.474683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067643904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.474699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.474706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.474730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067643904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763378.474748:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763378.474867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.474892:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa6300 x1876702067644032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.474909:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067644032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.474921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.474925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.474930:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763378.474938:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763378.474969:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.474982:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa4000 x1876702067644160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.474997:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067644160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.475016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.475020:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.475024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763378.475066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763378.475090:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.475103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa6d80 x1876702067644288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.475117:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067644288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.475128:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.475131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.475135:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763378.475143:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763378.475163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.475175:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa7100 x1876702067644416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.475199:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067644416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.475209:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.475212:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.475217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763378.475225:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067644544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763378.475246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.475258:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa5500 x1876702067644544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.475272:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067644544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.475281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.475285:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.475288:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067644544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763378.475295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067644672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763378.475341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763378.475359:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763378/real 1789763378] req@ffff936010aa4a80 x1876702067644672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763378.475378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067644672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763378.475388:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763378.475392:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763378.475396:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067644672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763379.242026:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763380.266090:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763381.289991:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763382.314076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763383.338076:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763383.593919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.593932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763383.593939:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.593956:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:3.0:1789763383.593977:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.593991:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763383.594061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067644800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763383.594117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.594123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763383.594125:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594131:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:3.0:1789763383.594147:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.594152:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763383.594167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763383.594185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067644928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763383.594199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.594208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763383.594211:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594217:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:3.0:1789763383.594222:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.594227:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763383.594232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763383.594246:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067645056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763383.594260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763383.594266:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067644800:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763383.594267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763383.594274:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594329:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:3.0:1789763383.594338:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.594346:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763383.594358:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763383.594386:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067645184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763383.594406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.594424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:1.0:1789763383.594427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763383.594429:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594442:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:3.0:1789763383.594451:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.594458:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763383.594468:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763383.594492:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067645312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763383.594504:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa4380 x1876702067644800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763383.594529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.594539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763383.594545:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067644800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763383.594547:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594570:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00080000:1.0:1789763383.594575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763383.594581:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763383.594594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763383.594608:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763383.594610:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067644800:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763383.594627:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763383.594631:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763383.594659:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067645440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763383.594675:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763383.594678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763383.594686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763383.594690:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763383.594699:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 851 00000100:00100000:1.0:1789763383.594700:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa5180 x1876702067644928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763383.594705:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763383.594717:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763383.594733:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763383.594740:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067644928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.594756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763383.594757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067645568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763383.594769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.594780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067644928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763383.594901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763383.594941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763383.594959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa7b80 x1876702067645056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763383.594976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067645056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.594986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763383.594990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.594995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763383.595002:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763383.595024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763383.595037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa7800 x1876702067645184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763383.595051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067645184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.595061:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763383.595064:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.595068:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763383.595074:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763383.595095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763383.595111:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa4a80 x1876702067645312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763383.595128:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067645312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.595152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763383.595156:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.595160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763383.595167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067645440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763383.595189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763383.595204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa5500 x1876702067645440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763383.595218:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067645440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.595228:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763383.595232:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.595235:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067645440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763383.595242:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067645568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763383.595262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763383.595274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763383/real 1789763383] req@ffff936010aa7100 x1876702067645568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763383.595315:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067645568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763383.595333:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763383.595336:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763383.595341:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067645568:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763384.362003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763385.386018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763386.410080:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763387.433949:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763388.458050:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763388.713990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714010:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763388.714017:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714036:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714055:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714069:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763388.714126:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067645696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763388.714175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763388.714183:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714189:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714194:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714200:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714214:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763388.714228:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067645824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763388.714265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763388.714277:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714286:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714294:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714300:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714308:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763388.714326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067645952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763388.714343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763388.714356:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763388.714359:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067645696:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763388.714368:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714382:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714391:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714397:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763388.714415:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067646080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763388.714428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763388.714438:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714443:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714480:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714486:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714493:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763388.714500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763388.714519:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067646208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763388.714538:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763388.714545:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714550:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714554:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714562:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714568:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763388.714585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067646336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763388.714601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef7100 x1876702067645696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763388.714613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763388.714627:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763388.714634:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763388.714656:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 856 00000100:00080000:1.0:1789763388.714665:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763388.714676:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763388.714685:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763388.714704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067645696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763388.714705:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067646464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763388.714868:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.714878:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.714886:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067645696:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763388.714902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763388.714967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.714990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef6680 x1876702067645824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067645824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715025:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763388.715066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763388.715090:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.715103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef5f80 x1876702067645952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715118:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067645952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715127:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067645952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763388.715142:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763388.715164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.715177:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef6a00 x1876702067646080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715191:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067646080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715209:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763388.715224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763388.715280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.715297:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef5500 x1876702067646208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715311:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067646208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715321:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715328:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763388.715336:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067646336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763388.715361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.715374:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef4e00 x1876702067646336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067646336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715396:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067646336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763388.715411:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067646464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763388.715433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763388.715445:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763388/real 1789763388] req@ffff936010ef6300 x1876702067646464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763388.715458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067646464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763388.715468:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763388.715471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763388.715475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067646464:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763389.482097:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763390.505965:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763391.530118:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763392.553943:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763393.578002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763393.833984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.833994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763393.833998:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763393.834009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763393.834024:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763393.834057:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067646592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763393.834092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763393.834096:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763393.834119:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834126:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763393.834133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763393.834156:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763393.834187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067646592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763393.834195:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067646720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763393.834270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763393.834277:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763393.834282:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834289:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763393.834296:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763393.834305:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763393.834327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763393.834331:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067646848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763393.834353:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763393.834363:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763393.834370:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834378:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763393.834378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020dad500 x1876702067646592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763393.834391:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763393.834404:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763393.834415:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067646592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763393.834424:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067646976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763393.834439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.834452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067646592:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763393.834466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834477:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763393.834479:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763393.834485:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834488:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763393.834492:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763393.834497:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763393.834503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763393.834512:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067647104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763393.834526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763393.834539:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763393.834539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763393.834552:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763393.834561:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020daf100 x1876702067646720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763393.834576:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763393.834588:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067646720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763393.834589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763393.834611:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763393.834618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067647232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763393.834622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646720:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763393.834642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763393.834652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763393.834655:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763393.834662:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763393.834663:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 861 00000100:00080000:0.0:1789763393.834672:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763393.834680:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763393.834687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763393.834702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067647360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763393.834732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763393.834762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020dac700 x1876702067646848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763393.834787:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067646848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763393.834797:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.834801:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834805:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763393.834813:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763393.834868:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763393.834895:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020dad880 x1876702067646976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763393.834903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067646976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763393.834909:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.834911:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067646976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763393.834924:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763393.834936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763393.834943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020daca80 x1876702067647104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763393.834950:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067647104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763393.834959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.834962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763393.834967:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067647232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763393.834977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763393.834983:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020dac000 x1876702067647232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763393.834990:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067647232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763393.834995:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.834997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.834999:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067647232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763393.835002:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067647360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763393.835012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763393.835018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763393/real 1789763393] req@ffff936020dafb80 x1876702067647360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763393.835025:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067647360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763393.835030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763393.835032:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763393.835034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067647360:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763394.602043:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763395.626067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763396.650100:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763397.674087:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763398.698038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763398.954034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763398.954049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763398.954056:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954071:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954088:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954098:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763398.954149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067647488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763398.954217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763398.954219:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954225:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954237:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763398.954251:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763398.954269:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067647616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763398.954291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067647488:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763398.954306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763398.954312:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954318:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954322:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954328:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763398.954335:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763398.954354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067647744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954367:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763398.954372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763398.954375:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954380:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954384:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954390:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763398.954395:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763398.954411:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067647872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763398.954431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763398.954434:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954441:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954445:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954451:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763398.954459:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647616:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789763398.954464:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763398.954489:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067648000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763398.954510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763398.954516:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954523:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954527:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763398.954529:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647744:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763398.954537:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763398.954548:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763398.954569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067648128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763398.954581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763398.954584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647872:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789763398.954594:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763398.954600:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763398.954608:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 866 00000100:00080000:0.0:1789763398.954615:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763398.954620:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763398.954629:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763398.954642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763398.954652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067648256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763398.954689:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067648128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763398.954723:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067648256:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763399.722114:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763400.746031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763401.770099:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763402.026252:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763402.026309:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763402.026351:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026400:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026412:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026422:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026431:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026443:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763402.026453:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763402.026563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020daca80 x1876702067648256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.026623:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067648256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.026649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.026659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.026669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067648256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763402.026687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020dafb80 x1876702067648000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.026716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067648000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.026734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.026742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.026750:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763402.026764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020dac000 x1876702067648128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.026791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067648128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.026880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.026890:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.026898:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067648128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763402.026915:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020daed80 x1876702067647616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.026945:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067647616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.026962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.026971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.026981:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763402.027000:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020dae680 x1876702067647744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.027026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067647744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.027043:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.027050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.027057:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763402.027071:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020dac380 x1876702067647872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.027098:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067647872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.027115:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.027120:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.027127:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067647872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763402.027178:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763398/real 1789763402] req@ffff936020daf480 x1876702067647488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763402.027208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067647488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763402.027225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763402.027232:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763402.027239:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067647488:192.168.203.137@tcp:250:kworker.0 00010000:00010000:1.0:1789763402.794014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763403.818093:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763404.073892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.073904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763404.073909:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763404.073922:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.073937:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.073959:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763404.074024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067648384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763404.074064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763404.074074:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763404.074080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074087:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.074094:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763404.074109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763404.074120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067648384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763404.074154:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067648512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763404.074179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763404.074185:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763404.074190:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074194:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.074199:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763404.074205:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763404.074219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067648640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763404.074238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763404.074248:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763404.074251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763404.074258:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074267:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.074274:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763404.074282:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763404.074318:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067648768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763404.074331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074337:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763404.074341:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763404.074349:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020dad880 x1876702067648384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763404.074352:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074363:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.074392:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763404.074401:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763404.074416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067648384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763404.074422:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067648896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763404.074450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763404.074454:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074463:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763404.074464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763404.074469:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763404.074474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067648384:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763404.074499:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074507:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763404.074516:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648512:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763404.074517:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763404.074532:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763404.074557:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067649024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763404.074579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763404.074596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763404.074603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763404.074609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020dac700 x1876702067648512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763404.074610:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763404.074624:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 871 00000100:00080000:0.0:1789763404.074636:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763404.074648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763404.074649:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067648512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763404.074658:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763404.074673:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.074682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.074688:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763404.074697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067649152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763404.074704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763404.074751:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763404.074775:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020daf100 x1876702067648640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763404.074793:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067648640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763404.074858:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.074863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.074867:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763404.074906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763404.074938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763404.074950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020dad500 x1876702067648768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763404.074966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067648768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763404.074977:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.074982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.074986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763404.075000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763404.075027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763404.075042:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020dadf80 x1876702067648896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763404.075066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067648896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763404.075077:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.075081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.075085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067648896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763404.075114:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763404.075160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763404.075176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020daf800 x1876702067649024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763404.075193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067649024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763404.075203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.075208:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.075212:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763404.075220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763404.075245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763404.075261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763404/real 1789763404] req@ffff936020dae300 x1876702067649152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763404.075276:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067649152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763404.075295:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763404.075299:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763404.075304:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649152:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763404.842043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763405.866070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763406.890075:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763407.914032:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763408.938042:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763409.193972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.193982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763409.193987:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.193999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194013:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763409.194072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067649280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763409.194117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763409.194123:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194127:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194134:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763409.194171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067649408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763409.194181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763409.194186:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194192:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194197:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763409.194198:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067649280:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763409.194202:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194210:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763409.194219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067649536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763409.194233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763409.194238:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194241:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194244:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194246:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194250:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763409.194267:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067649664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763409.194275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763409.194280:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194282:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194285:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194288:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194292:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763409.194300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067649792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763409.194309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763409.194312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763409.194318:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194323:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194336:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194340:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194363:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763409.194376:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067649920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763409.194388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763409.194392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763409.194403:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763409.194408:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 877 00000100:00080000:3.0:1789763409.194414:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763409.194420:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763409.194426:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763409.194434:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa6d80 x1876702067649280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763409.194449:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067650048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763409.194496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067649280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194527:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067649280:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763409.194584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763409.194614:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194632:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa4000 x1876702067649408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067649408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194661:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194664:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763409.194672:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763409.194692:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa6300 x1876702067649536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067649536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194717:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194720:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194722:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763409.194726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763409.194751:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa6680 x1876702067649664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194767:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067649664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194773:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194779:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763409.194783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763409.194847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa5880 x1876702067649792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194867:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067649792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194874:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194877:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067649792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763409.194883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763409.194916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa7480 x1876702067649920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194934:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067649920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194941:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194946:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067649920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763409.194950:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763409.194963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763409.194971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763409/real 1789763409] req@ffff936010aa5c00 x1876702067650048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763409.194979:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067650048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763409.194985:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763409.194987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763409.194989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650048:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763409.962025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763410.986105:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763412.010019:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763413.034095:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763414.058042:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763414.313978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.313993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763414.313999:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314015:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:0.0:1789763414.314031:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314043:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763414.314138:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067650176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763414.314189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763414.314202:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314207:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:0.0:1789763414.314213:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314220:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314236:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763414.314254:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067650304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763414.314271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067650176:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763414.314273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763414.314292:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314299:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:0.0:1789763414.314303:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314307:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314314:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763414.314332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067650432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763414.314344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763414.314351:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314357:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:0.0:1789763414.314379:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314384:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314392:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763414.314396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763414.314413:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067650560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763414.314429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763414.314435:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314439:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:0.0:1789763414.314444:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314450:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314457:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763414.314468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020dad180 x1876702067650176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763414.314477:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067650688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763414.314515:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763414.314527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067650176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763414.314529:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314550:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 882 00000100:00080000:1.0:1789763414.314557:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763414.314562:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763414.314568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763414.314580:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314590:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763414.314599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067650176:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763414.314638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067650816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763414.314643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650304:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763414.314676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763414.314684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763414.314689:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763414.314699:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 882 00000800:00000100:1.0:1789763414.314709:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763414.314710:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763414.314729:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763414.314741:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763414.314753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020daea00 x1876702067650304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763414.314770:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067650944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763414.314785:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067650304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.314845:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.314852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.314859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763414.314874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763414.314909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763414.314925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020dace00 x1876702067650432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763414.314941:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067650432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.314951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.314955:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.314959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763414.314966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763414.314988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763414.315000:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020dadc00 x1876702067650560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763414.315014:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067650560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.315025:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.315029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.315034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763414.315040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763414.315094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763414.315119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020dae300 x1876702067650688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763414.315144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067650688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.315161:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.315168:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.315174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067650688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763414.315202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763414.315233:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763414.315248:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020daf800 x1876702067650816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763414.315264:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067650816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.315274:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.315278:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.315283:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763414.315290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763414.315311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763414.315323:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763414/real 1789763414] req@ffff936020dadf80 x1876702067650944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763414.315336:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067650944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763414.315346:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763414.315350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763414.315354:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067650944:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763415.082129:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763416.106084:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763417.130087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763418.154049:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763419.178045:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763419.434110:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763419.434129:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434142:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434158:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434168:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763419.434225:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067651072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763419.434273:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434277:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434281:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434285:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763419.434298:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763419.434312:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067651200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434328:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763419.434330:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434334:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434337:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434345:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763419.434348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067651072:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763419.434351:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763419.434368:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067651328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434379:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763419.434385:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434389:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434393:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434396:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763419.434402:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763419.434415:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067651456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763419.434432:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434435:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434438:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434441:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763419.434445:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763419.434459:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067651584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763419.434472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763419.434474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763419.434481:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434486:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434491:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434495:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763419.434500:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763419.434510:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067651712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763419.434519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763419.434522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763419.434524:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763419.434528:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 887 00000100:00080000:3.0:1789763419.434533:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763419.434536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763419.434542:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763419.434564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067651840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763419.434566:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa5f80 x1876702067651072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.434622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067651072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.434641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.434649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.434657:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067651072:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763419.434692:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763419.434733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.434761:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa4e00 x1876702067651200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.434775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067651200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.434783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.434787:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.434790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763419.434901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763419.434926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.434958:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa4700 x1876702067651328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.434973:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067651328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.434984:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.434988:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.434993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763419.435000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763419.435095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.435108:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa6a00 x1876702067651456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.435120:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067651456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.435129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.435132:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.435152:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763419.435159:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763419.435182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.435194:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa5c00 x1876702067651584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.435208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067651584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.435218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.435221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.435224:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067651584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763419.435230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067651712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763419.435251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.435262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa7480 x1876702067651712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.435276:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067651712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.435286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.435290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.435294:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067651712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763419.435300:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067651840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763419.435319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763419.435330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763419/real 1789763419] req@ffff936010aa5880 x1876702067651840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763419.435364:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067651840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763419.435379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763419.435384:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763419.435390:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067651840:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763420.202174:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763421.226057:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763422.250007:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763423.274075:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763424.298135:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763424.554056:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763424.554070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763424.554076:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554103:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554121:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554132:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763424.554211:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067651968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763424.554255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763424.554260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763424.554263:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554268:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554273:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554277:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554291:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763424.554308:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067652096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763424.554325:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763424.554335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067651968:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763424.554335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763424.554341:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554356:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554361:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554365:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554371:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763424.554398:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067652224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763424.554411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763424.554415:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763424.554418:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554425:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554462:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554468:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763424.554476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554477:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763424.554500:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067652352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763424.554513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763424.554518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763424.554520:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554525:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554529:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554534:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554541:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763424.554554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef6d80 x1876702067651968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763424.554560:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067652480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763424.554601:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763424.554606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763424.554609:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554617:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00100000:0.0:1789763424.554621:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067651968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763424.554622:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554635:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554652:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763424.554667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789763424.554670:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067652608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763424.554686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763424.554699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763424.554701:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067651968:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763424.554711:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763424.554717:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763424.554724:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 892 00000100:00080000:1.0:1789763424.554728:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763424.554732:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763424.554738:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763424.554750:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067652736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763424.554758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763424.554894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.554917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef7480 x1876702067652096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.554935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067652096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.554956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.554960:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.554965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763424.554973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763424.555033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.555050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef7b80 x1876702067652224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.555065:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067652224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.555075:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.555079:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.555083:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763424.555091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763424.555125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.555137:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef7800 x1876702067652352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.555151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067652352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.555161:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.555165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.555170:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763424.555176:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763424.555198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.555210:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef4380 x1876702067652480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.555224:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067652480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.555233:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.555236:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.555240:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763424.555246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067652608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763424.555267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.555279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef4000 x1876702067652608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.555292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067652608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.555301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.555305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.555308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067652608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763424.555331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067652736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763424.555352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763424.555365:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763424/real 1789763424] req@ffff936010ef5c00 x1876702067652736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763424.555389:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067652736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763424.555399:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763424.555402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763424.555406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067652736:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763425.322101:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763426.346122:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763427.370070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763428.394135:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763429.418099:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763429.674063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763429.674084:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674100:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674118:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763429.674130:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763429.674194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067652864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763429.674246:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674252:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674264:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763429.674269:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674284:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763429.674300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067652992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763429.674332:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674338:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674342:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763429.674352:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067652864:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763429.674353:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674360:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763429.674375:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067653120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674396:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763429.674398:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674403:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674407:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763429.674411:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674417:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763429.674439:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067653248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763429.674468:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674476:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674485:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763429.674499:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763429.674500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674533:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763429.674556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067653376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763429.674578:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674589:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00100000:0.0:1789763429.674592:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa6680 x1876702067652864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763429.674596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763429.674611:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674625:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763429.674645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067652864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763429.674647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067653504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763429.674711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763429.674718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763429.674720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763429.674721:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763429.674736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 897 00000100:00080000:3.0:1789763429.674742:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763429.674744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763429.674751:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763429.674761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763429.674768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067652864:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763429.674778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067653632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763429.674866:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763429.674921:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.674971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa6300 x1876702067652992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.674991:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067652992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675011:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675016:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067652992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763429.675058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763429.675086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.675101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa4000 x1876702067653120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.675115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067653120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675125:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675132:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763429.675139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763429.675160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.675173:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa6d80 x1876702067653248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.675186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067653248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675196:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675199:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675204:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763429.675210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763429.675230:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.675242:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa7100 x1876702067653376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.675255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067653376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675267:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675271:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763429.675277:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067653504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763429.675300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.675312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa5500 x1876702067653504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.675325:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067653504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675338:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067653504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763429.675348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067653632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763429.675368:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763429.675380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763429/real 1789763429] req@ffff936010aa4a80 x1876702067653632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763429.675393:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067653632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763429.675402:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763429.675406:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763429.675410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067653632:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763430.442163:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763431.466079:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763432.490077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763433.513927:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763434.538036:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763434.794007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763434.794027:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794042:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794057:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794067:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763434.794130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067653760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763434.794176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763434.794184:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794190:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794194:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794198:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794210:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763434.794224:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067653888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763434.794239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763434.794247:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794252:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794256:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794260:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794266:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763434.794278:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067654016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763434.794291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067653760:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763434.794296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763434.794312:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794318:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794321:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794325:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794332:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763434.794346:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067654144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763434.794358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763434.794365:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794370:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794398:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794404:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794410:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763434.794420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763434.794429:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067654272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763434.794443:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763434.794451:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794457:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794461:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794465:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794470:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763434.794484:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067654400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763434.794496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763434.794504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763434.794505:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020dad500 x1876702067653760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763434.794507:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763434.794521:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 902 00000100:00080000:0.0:1789763434.794528:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763434.794538:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763434.794550:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763434.794571:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067653760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763434.794580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067654528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763434.794668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.794679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.794690:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067653760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763434.794707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763434.794752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.794776:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020daf100 x1876702067653888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.794877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067653888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.794897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.794932:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.794940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067653888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763434.794992:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763434.795033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.795058:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020dac700 x1876702067654016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.795083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067654016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.795108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.795115:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.795122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763434.795135:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763434.795173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.795195:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020dad880 x1876702067654144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.795219:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067654144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.795237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.795243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.795250:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763434.795262:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763434.795298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.795317:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020daf480 x1876702067654272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.795337:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067654272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.795353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.795359:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.795365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763434.795376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067654400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763434.795421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.795440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020dac380 x1876702067654400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.795461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067654400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.795476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.795481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.795487:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067654400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763434.795498:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067654528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763434.795536:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763434.795558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763434/real 1789763434] req@ffff936020dae680 x1876702067654528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763434.795581:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067654528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763434.795595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763434.795601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763434.795607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067654528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763435.562069:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763436.586041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763437.610128:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763438.634084:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763439.658052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763439.913981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.913991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763439.913996:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914007:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763439.914075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067654656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763439.914130:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914135:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914140:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914144:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763439.914188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067654784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763439.914202:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914205:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914207:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914210:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914215:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763439.914223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067654656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763439.914228:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067654912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763439.914253:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914257:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914259:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914264:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763439.914284:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daca80 x1876702067655040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914293:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763439.914294:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914297:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914310:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914314:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914320:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763439.914325:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763439.914337:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067655168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914347:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763439.914351:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914354:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00080000:0.0:1789763439.914357:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914360:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914365:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763439.914381:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067655296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763439.914386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763439.914388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763439.914389:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763439.914395:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 907 00000100:00100000:1.0:1789763439.914397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020daed80 x1876702067654656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763439.914399:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763439.914409:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763439.914420:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763439.914437:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067655424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763439.914450:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067654656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914465:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914476:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067654656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763439.914485:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763439.914522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.914536:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020dac000 x1876702067654784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.914549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067654784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914557:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914571:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763439.914611:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763439.914630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.914640:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020dafb80 x1876702067654912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.914661:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067654912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067654912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763439.914688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763439.914705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.914715:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020daca80 x1876702067655040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.914726:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daca80 x1876702067655040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763439.914745:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763439.914761:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.914771:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020dae680 x1876702067655168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.914781:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067655168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914791:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914834:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763439.914841:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067655296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763439.914862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.914904:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020dac380 x1876702067655296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.914932:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067655296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.914946:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.914951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.914957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067655296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763439.914966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067655424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763439.914994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763439.915006:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763439/real 1789763439] req@ffff936020daf480 x1876702067655424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763439.915018:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067655424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763439.915026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763439.915030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763439.915033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067655424:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763440.682055:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763441.706052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763442.730074:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763443.754062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763444.778089:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763445.034036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763445.034055:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034071:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034085:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034094:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763445.034147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067655552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034197:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763445.034200:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034205:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034210:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034216:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763445.034245:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067655680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034261:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763445.034268:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034274:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034287:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763445.034290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067655552:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763445.034305:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763445.034328:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067655808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034345:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763445.034349:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034355:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034359:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034364:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763445.034382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067655936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763445.034399:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034404:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034423:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034428:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034434:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763445.034450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763445.034452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067656064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034472:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763445.034479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763445.034484:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034490:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034499:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034505:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034512:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763445.034530:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067656192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763445.034543:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763445.034547:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef5880 x1876702067655552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763445.034550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763445.034557:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763445.034579:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 912 00000100:00080000:1.0:1789763445.034590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763445.034595:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763445.034602:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763445.034624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067656320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763445.034633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067655552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.034693:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.034705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.034713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067655552:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763445.034756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763445.034794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.034886:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef4a80 x1876702067655680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.034904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067655680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.034915:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.034920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.034925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763445.034935:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763445.034965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.034980:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef5180 x1876702067655808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.034995:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067655808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.035005:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.035009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.035013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763445.035021:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763445.035062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.035076:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef4700 x1876702067655936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.035091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067655936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.035101:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.035105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.035108:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067655936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763445.035116:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763445.035138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.035151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef5c00 x1876702067656064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.035164:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067656064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.035174:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.035178:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.035181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763445.035188:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067656192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763445.035210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.035222:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef4000 x1876702067656192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.035245:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067656192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.035254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.035258:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.035261:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067656192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763445.035268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067656320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763445.035289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763445.035301:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763445/real 1789763445] req@ffff936010ef4380 x1876702067656320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763445.035314:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067656320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763445.035323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763445.035327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763445.035333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067656320:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763445.802075:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763446.826081:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763447.850175:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763448.873972:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763449.898053:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763450.154034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763450.154056:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154086:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154104:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154117:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763450.154184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067656448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763450.154223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763450.154231:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154241:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154246:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154260:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763450.154275:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067656576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763450.154290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154295:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763450.154297:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154303:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154307:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154311:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154319:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763450.154329:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067656448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763450.154336:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067656704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763450.154353:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763450.154358:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154367:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154371:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154375:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154380:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763450.154396:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067656832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763450.154408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763450.154415:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154420:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154426:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154431:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154436:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763450.154454:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067656960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763450.154474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763450.154482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763450.154496:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763450.154507:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154515:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154524:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154536:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763450.154557:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa7800 x1876702067656448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763450.154565:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067657088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763450.154607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763450.154612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763450.154616:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763450.154619:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067656448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763450.154628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 917 00000100:00080000:3.0:1789763450.154640:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763450.154653:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763450.154667:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763450.154676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763450.154686:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067657216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763450.154688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.154704:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067656448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763450.154730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763450.154766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.154785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa7b80 x1876702067656576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.154881:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067656576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.154898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.154905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.154910:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763450.154949:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763450.154981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.154997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa5180 x1876702067656704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.155011:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067656704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.155022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.155026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.155029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763450.155037:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763450.155059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.155072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa4380 x1876702067656832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.155086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067656832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.155096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.155100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.155103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763450.155109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763450.155129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.155141:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa4a80 x1876702067656960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.155155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067656960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.155164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.155167:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.155171:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067656960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763450.155177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763450.155199:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.155211:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa5500 x1876702067657088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.155226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067657088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.155235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.155239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.155245:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763450.155255:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763450.155291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763450.155313:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763450/real 1789763450] req@ffff936010aa7100 x1876702067657216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763450.155338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067657216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763450.155354:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763450.155358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763450.155363:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657216:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763450.922057:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763451.946050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763452.969985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763453.994066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763455.018136:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763455.273914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763455.273932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763455.273941:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.273957:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.273997:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763455.274093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067657344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763455.274161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763455.274167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763455.274170:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274176:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.274183:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274190:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274206:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763455.274226:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067657472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763455.274242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763455.274247:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067657344:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763455.274249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763455.274256:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274269:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.274274:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274278:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274284:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763455.274303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067657600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763455.274317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763455.274322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763455.274325:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274330:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.274336:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274340:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274345:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763455.274363:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067657728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763455.274384:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763455.274389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763455.274399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763455.274404:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274415:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.274425:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274433:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274444:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763455.274469:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067657856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763455.274483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763455.274491:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa6d80 x1876702067657344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763455.274492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763455.274499:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274514:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00080000:3.0:1789763455.274525:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274546:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763455.274548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067657344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763455.274580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067657984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763455.274583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.274600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.274613:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067657344:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763455.274645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763455.274650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763455.274653:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763455.274662:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 923 00000100:00100000:0.0:1789763455.274668:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657472:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763455.274673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763455.274684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763455.274693:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763455.274713:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067658112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763455.274720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.274788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa4000 x1876702067657472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.274877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067657472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.274894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.274901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.274907:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763455.274920:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763455.274965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.274989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa6300 x1876702067657600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.275014:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067657600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.275031:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.275037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.275044:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763455.275057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763455.275099:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.275120:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa6680 x1876702067657728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.275142:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067657728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.275159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.275165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.275172:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763455.275183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763455.275226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.275251:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa5880 x1876702067657856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.275277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067657856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.275293:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.275298:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.275302:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067657856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763455.275310:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763455.275346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.275369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa7480 x1876702067657984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.275395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067657984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.275413:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.275420:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.275426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067657984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763455.275469:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067658112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763455.275500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763455.275515:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763455/real 1789763455] req@ffff936010aa5c00 x1876702067658112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763455.275530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067658112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763455.275541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763455.275545:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763455.275549:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067658112:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763456.042076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763457.066067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763458.090029:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763459.114045:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763460.138059:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763460.393899:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.393913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763460.393920:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.393934:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.393950:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.393961:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763460.394018:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067658240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763460.394056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.394061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763460.394064:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.394069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763460.394095:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763460.394112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067658368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763460.394127:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067658240:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763460.394130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.394142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763460.394145:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.394152:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394156:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394161:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763460.394167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763460.394183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067658496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763460.394196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.394201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763460.394204:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.394209:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394215:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394220:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763460.394227:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763460.394245:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067658624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763460.394247:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763460.394265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.394271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763460.394275:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.394282:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394287:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394291:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763460.394297:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763460.394314:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067658752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763460.394342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763460.394347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763460.394351:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763460.394356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa6a00 x1876702067658240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763460.394361:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394372:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394397:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763460.394406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067658240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763460.394411:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763460.394438:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067658880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763460.394442:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.394457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763460.394465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763460.394472:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067658240:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763460.394474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763460.394483:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763460.394492:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 928 00000100:00080000:3.0:1789763460.394497:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763460.394512:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763460.394522:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763460.394525:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763460.394562:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067659008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763460.394581:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.394612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa4700 x1876702067658368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.394631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067658368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.394643:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.394648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.394652:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763460.394659:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763460.394682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.394695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa4e00 x1876702067658496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.394709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067658496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.394753:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.394758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.394762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763460.394771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763460.394838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.394855:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa5f80 x1876702067658624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.394870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067658624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.394881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.394885:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.394889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763460.394896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763460.394920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.394933:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa5c00 x1876702067658752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.394947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067658752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.394956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.394959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.394963:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067658752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763460.394969:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067658880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763460.394990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.395002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa7480 x1876702067658880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.395016:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067658880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.395026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.395029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.395033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067658880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763460.395058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763460.395080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763460.395092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763460/real 1789763460] req@ffff936010aa5880 x1876702067659008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763460.395106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067659008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763460.395121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763460.395127:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763460.395134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763461.161970:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763462.186042:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763463.210091:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763464.234104:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763465.258055:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763465.513925:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.513948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763465.513952:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.513963:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.513975:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.513985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763465.514033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067659136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514075:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763465.514078:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514084:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.514090:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514095:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514107:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763465.514120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067659264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514129:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763465.514134:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514138:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.514142:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514146:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514150:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763465.514159:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067659392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763465.514173:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514176:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.514182:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514186:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514189:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763465.514199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067659520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763465.514213:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514226:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00100000:0.0:1789763465.514230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067659136:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763465.514232:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514271:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514281:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763465.514300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067659648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514333:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763465.514334:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514338:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.514374:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514378:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514400:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763465.514425:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067659776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763465.514436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763465.514440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763465.514444:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763465.514453:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 933 00000100:00080000:1.0:1789763465.514460:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763465.514470:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763465.514479:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763465.514489:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763465.514498:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067659904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763465.514569:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763465.514617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763465.514646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763465.514702:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763465.514740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763466.281992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763467.306067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763468.330024:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763468.586123:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789763468.586178:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763468.586199:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763468.586248:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763468.586262:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763468.586274:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763468.586285:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789763468.586294:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789763468.586303:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef7800 x1876702067659136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789763468.586306:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789763468.586339:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067659136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586352:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586362:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067659136:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763468.586370:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef7b80 x1876702067659264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586382:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067659264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586390:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586394:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763468.586404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef7480 x1876702067659392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067659392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586431:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763468.586467:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef6d80 x1876702067659520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067659520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586490:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586494:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586498:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763468.586506:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef6300 x1876702067659648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067659648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067659648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763468.586541:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef4e00 x1876702067659776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586553:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067659776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586566:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763468.586576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763465/real 1789763468] req@ffff936010ef5500 x1876702067659904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763468.586587:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067659904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763468.586594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763468.586597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763468.586600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067659904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763469.354071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763470.378009:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763470.634056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763470.634078:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634094:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:3.0:1789763470.634111:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634134:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763470.634188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067660032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763470.634241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763470.634254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634271:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:3.0:1789763470.634279:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634290:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763470.634309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763470.634338:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067660160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763470.634350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067660032:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763470.634373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763470.634387:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634397:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:3.0:1789763470.634405:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634413:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763470.634425:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763470.634452:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067660288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763470.634472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763470.634482:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634494:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 938 00000800:00000100:0.0:1789763470.634498:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763470.634504:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634517:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763470.634529:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763470.634553:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067660416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763470.634574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763470.634586:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634594:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:3.0:1789763470.634602:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634612:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763470.634621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa6680 x1876702067660032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763470.634628:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763470.634689:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067660544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763470.634711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067660032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763470.634717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763470.634745:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763470.634755:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763470.634760:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:0.0:1789763470.634772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763470.634775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763470.634787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067660032:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763470.634793:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763470.634865:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763470.634890:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067660672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763470.634908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660160:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763470.634916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763470.634929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763470.634933:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763470.634944:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 938 00000100:00080000:3.0:1789763470.634953:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763470.634973:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763470.634986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763470.634993:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763470.635024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067660800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763470.635030:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa6300 x1876702067660160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635097:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067660160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635116:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635124:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635133:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763470.635146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763470.635192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763470.635218:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa4000 x1876702067660288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635259:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067660288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635285:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763470.635305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763470.635349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763470.635375:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa6d80 x1876702067660416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067660416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635452:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763470.635463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763470.635506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763470.635536:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa7100 x1876702067660544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067660544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067660544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763470.635632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067660672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763470.635705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763470.635730:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa5500 x1876702067660672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635753:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067660672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635839:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067660672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763470.635853:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067660800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763470.635898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763470.635917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763470/real 1789763470] req@ffff936010aa4a80 x1876702067660800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763470.635933:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067660800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763470.635943:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763470.635947:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763470.635951:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067660800:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763471.402077:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763472.426055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763473.450056:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763474.474069:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763475.498085:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763475.753957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.753967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763475.753972:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.753985:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.753999:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754008:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763475.754053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067660928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763475.754095:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.754100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754107:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754118:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763475.754133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067661056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763475.754153:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067660928:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763475.754155:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.754161:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754165:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754176:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763475.754198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067661184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763475.754213:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.754218:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754220:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754224:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754228:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763475.754238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067661312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763475.754252:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.754255:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754259:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754262:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754267:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763475.754273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763475.754285:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067661440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763475.754303:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763475.754306:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754309:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754312:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763475.754331:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa4380 x1876702067660928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763475.754333:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067661568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763475.754372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763475.754376:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763475.754379:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763475.754385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067660928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763475.754385:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 943 00000100:00080000:3.0:1789763475.754394:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763475.754402:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763475.754410:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763475.754411:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754425:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067660928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763475.754429:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067661696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763475.754463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763475.754498:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.754520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa5180 x1876702067661056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.754533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067661056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.754542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754546:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763475.754559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763475.754580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.754590:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa7b80 x1876702067661184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.754621:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067661184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.754630:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763475.754682:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763475.754716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.754729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa7800 x1876702067661312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.754745:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067661312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.754754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754761:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763475.754767:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763475.754825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.754846:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa4a80 x1876702067661440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.754865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067661440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.754874:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754879:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754884:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763475.754894:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067661568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763475.754938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.754952:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa5500 x1876702067661568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.754966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067661568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.754973:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.754976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.754978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067661568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763475.754983:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067661696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763475.755007:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763475.755018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763475/real 1789763475] req@ffff936010aa7100 x1876702067661696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763475.755037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067661696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763475.755045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763475.755049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763475.755052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067661696:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763476.522048:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763477.546073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763478.570046:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763479.593931:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763480.618049:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763480.874027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763480.874048:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763480.874061:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874076:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874085:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763480.874130:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067661824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763480.874172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763480.874180:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763480.874185:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874191:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874196:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.874209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763480.874227:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067661952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763480.874245:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067661824:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763480.874303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763480.874316:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763480.874329:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874337:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874346:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.874359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763480.874387:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067662080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763480.874410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763480.874430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763480.874440:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763480.874451:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874464:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.874482:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763480.874506:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067662208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763480.874523:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef6a00 x1876702067661824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763480.874531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763480.874560:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763480.874597:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874604:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874609:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.874621:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763480.874622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067661824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.874659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.874670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763480.874671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067662336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763480.874681:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067661824:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763480.874724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763480.874737:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763480.874745:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661952:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763480.874751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.874767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763480.874775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.874785:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763480.874864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.874889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef5f80 x1876702067661952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763480.874907:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067662464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763480.874915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067661952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763480.874987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763480.874996:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763480.875000:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763480.875005:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763480.875012:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 948 00000100:00080000:1.0:1789763480.875021:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763480.875023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763480.875030:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763480.875035:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763480.875052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067661952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763480.875060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067662592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763480.875075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763480.875130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.875151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef6680 x1876702067662080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763480.875172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067662080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.875185:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.875190:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763480.875195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763480.875206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763480.875231:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.875244:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef7100 x1876702067662208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763480.875279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067662208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.875295:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.875301:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763480.875308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763480.875320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763480.875383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.875398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef5500 x1876702067662336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763480.875430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067662336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.875440:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.875444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763480.875447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763480.875473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067662464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763480.875497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.875508:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef4e00 x1876702067662464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763480.875521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067662464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.875547:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.875551:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763480.875554:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067662464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763480.875560:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067662592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763480.875615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763480.875628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763480/real 1789763480] req@ffff936010ef6300 x1876702067662592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763480.875640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067662592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763480.875649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763480.875652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763480.875655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067662592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763481.642076:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763482.666038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763483.690053:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763484.714132:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763485.738041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763485.994019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763485.994039:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994053:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994071:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994082:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763485.994143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067662720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994193:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763485.994196:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994200:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994204:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994208:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763485.994223:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763485.994238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067662848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763485.994256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067662720:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763485.994259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763485.994265:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994274:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994277:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994281:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763485.994286:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763485.994301:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067662976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763485.994343:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994348:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994376:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994382:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763485.994390:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763485.994396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763485.994414:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067663104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994429:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994434:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763485.994436:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994440:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994445:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994450:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763485.994456:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763485.994473:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020dad880 x1876702067662720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763485.994476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067663232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763485.994593:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763485.994602:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067662720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763485.994605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00080000:0.0:1789763485.994619:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763485.994641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763485.994647:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763485.994654:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.994663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067662720:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763485.994677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067663360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763485.994691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763485.994696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763485.994699:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763485.994710:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 953 00000100:00100000:1.0:1789763485.994710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763485.994716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763485.994725:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763485.994732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763485.994749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067663488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763485.994763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.994790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020dac700 x1876702067662848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.994807:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067662848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.994850:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.994855:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.994859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763485.994868:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763485.994895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.994916:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020daf100 x1876702067662976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.994939:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067662976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.994955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.994961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.994968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067662976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763485.994980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763485.995014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.995028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020dad500 x1876702067663104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.995041:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067663104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.995052:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.995056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.995059:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763485.995065:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763485.995089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.995107:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020dadf80 x1876702067663232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.995131:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067663232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.995147:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.995154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.995160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763485.995171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067663360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763485.995205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.995220:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020daf800 x1876702067663360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.995234:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067663360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.995244:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.995248:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.995252:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067663360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763485.995274:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067663488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763485.995294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763485.995305:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763485/real 1789763485] req@ffff936020dae300 x1876702067663488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763485.995318:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067663488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763485.995353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763485.995358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763485.995362:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067663488:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763486.762038:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763487.786126:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763488.810119:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763489.834073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763490.858108:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763491.113894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.113904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763491.113908:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.113918:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.113927:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.113933:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763491.113967:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067663616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.113990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.113993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763491.113994:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.113997:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.113999:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.114001:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763491.114009:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763491.114017:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067663744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.114030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.114032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763491.114033:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067663616:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763491.114034:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.114041:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.114044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.114046:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763491.114049:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763491.114058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067663872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.114064:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.114067:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763491.114068:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.114070:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.114073:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.114075:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763491.114078:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763491.114084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067664000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.114090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.114093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763491.114094:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763491.114098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763491.114102:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.114105:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.114108:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763491.114112:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763491.114119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067664128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.114124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.114126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763491.114127:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.114130:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.114133:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763491.114138:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061dd500 x1876702067663616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763491.114142:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763491.114152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067664256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763491.114163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763491.114166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763491.114167:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763491.114170:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 958 00000100:00080000:2.0:1789763491.114173:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763491.114174:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067663616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763491.114180:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763491.114187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763491.114193:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763491.114197:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067664384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763491.114200:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114208:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067663616:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763491.114234:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763491.114252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061df480 x1876702067663744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114269:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067663744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114279:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763491.114287:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763491.114298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114304:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061df100 x1876702067663872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114311:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067663872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114316:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067663872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763491.114323:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763491.114333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114340:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061dd880 x1876702067664000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114346:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067664000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763491.114359:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763491.114370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114376:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061ddc00 x1876702067664128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114382:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067664128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114387:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114391:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763491.114400:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067664256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763491.114412:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061de300 x1876702067664256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067664256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114432:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067664256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763491.114437:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067664384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763491.114447:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763491.114454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763491/real 1789763491] req@ffff9360061ddf80 x1876702067664384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763491.114461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067664384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763491.114466:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763491.114467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763491.114469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067664384:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763491.882063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763492.905980:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763493.930074:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763494.954177:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763495.977993:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763496.233911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.233925:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763496.233931:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.233945:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.233959:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.233968:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763496.234018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067664512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763496.234055:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.234061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763496.234063:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.234068:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234072:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.234077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234088:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763496.234103:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067664640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763496.234118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.234122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763496.234125:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.234129:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234134:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.234139:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234145:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763496.234176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067664768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763496.234177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067664512:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763496.234220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.234224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763496.234226:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.234234:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234238:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.234244:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234250:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763496.234266:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067664896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763496.234277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.234283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763496.234285:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.234291:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234295:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.234300:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763496.234319:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067665024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763496.234334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763496.234383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763496.234391:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763496.234395:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763496.234409:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234417:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763496.234419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061dc700 x1876702067664512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763496.234441:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234481:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763496.234493:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067664512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763496.234511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067665152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763496.234520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763496.234534:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763496.234540:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.234559:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067664512:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763496.234561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763496.234577:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763496.234591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664640:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763496.234602:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 964 00000100:00080000:2.0:1789763496.234617:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763496.234629:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763496.234640:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763496.234659:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067665280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763496.234686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.234725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061dc380 x1876702067664640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.234744:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067664640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.234756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.234761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.234766:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763496.234875:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763496.234922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.234940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061ddf80 x1876702067664768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.234956:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067664768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.234967:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.234972:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.234976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763496.234984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763496.235009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.235022:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061de300 x1876702067664896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.235037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067664896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.235046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.235050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.235055:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067664896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763496.235062:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763496.235084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.235096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061ddc00 x1876702067665024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.235110:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067665024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.235119:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.235123:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.235127:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763496.235133:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067665152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763496.235154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.235166:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061dd880 x1876702067665152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.235181:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067665152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.235190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.235194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.235197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067665152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763496.235204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067665280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763496.235225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763496.235237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763496/real 1789763496] req@ffff9360061df100 x1876702067665280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763496.235250:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067665280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763496.235260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763496.235264:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763496.235268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067665280:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763497.002043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763498.026045:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763499.049962:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763500.074121:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763501.098064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763501.353924:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.353939:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763501.353946:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.353961:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.353980:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.353991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763501.354056:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067665408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763501.354107:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.354117:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354121:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763501.354135:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763501.354150:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067665536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354164:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763501.354171:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354176:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.354180:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763501.354190:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763501.354198:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067665408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763501.354210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067665664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763501.354234:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354239:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.354243:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354247:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763501.354253:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763501.354268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067665792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763501.354285:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354290:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.354295:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354299:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763501.354306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763501.354308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763501.354331:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067665920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354350:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763501.354352:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354357:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00080000:2.0:1789763501.354362:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354367:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763501.354376:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763501.354386:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061df480 x1876702067665408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763501.354395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067666048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763501.354446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763501.354453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763501.354456:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763501.354463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 969 00000100:00100000:1.0:1789763501.354471:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067665408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763501.354473:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763501.354499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763501.354500:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.354518:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:2.0:1789763501.354518:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763501.354533:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067665408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763501.354548:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067666176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763501.354550:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763501.354595:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.354618:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061dd500 x1876702067665536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.354644:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067665536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.354654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.354658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.354662:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763501.354697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763501.354719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.354732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061dca80 x1876702067665664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.354746:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067665664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.354756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.354759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.354763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763501.354770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763501.354790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.354849:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061df800 x1876702067665792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.354865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067665792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.354898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.354901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.354906:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763501.354913:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763501.354939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.354953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061dfb80 x1876702067665920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.354967:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067665920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.354995:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.355001:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.355007:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067665920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763501.355020:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763501.355062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.355086:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061de680 x1876702067666048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.355103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067666048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.355114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.355117:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.355123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763501.355130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763501.355157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763501.355170:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763501/real 1789763501] req@ffff9360061dc000 x1876702067666176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763501.355184:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067666176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763501.355193:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763501.355196:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763501.355200:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763502.122016:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763503.146062:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763504.170102:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763505.193999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763506.218086:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763506.473963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.473977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763506.473984:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.473998:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474014:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763506.474085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067666304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763506.474131:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474137:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474141:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474145:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474159:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763506.474174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067666432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763506.474203:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474208:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474212:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474217:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474224:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763506.474236:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067666560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763506.474254:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474259:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474264:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474268:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474275:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763506.474284:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067666304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763506.474295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067666688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474309:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763506.474315:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474326:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474330:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474336:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763506.474356:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067666816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763506.474397:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474403:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474417:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474423:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474431:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763506.474433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763506.474454:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067666944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763506.474470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763506.474475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763506.474477:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763506.474482:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 974 00000100:00080000:0.0:1789763506.474486:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763506.474492:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763506.474498:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763506.474512:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020dadc00 x1876702067666304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763506.474518:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067667072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763506.474572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067666304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.474631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.474639:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.474648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067666304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763506.474659:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763506.474693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.474709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020dace00 x1876702067666432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.474725:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067666432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.474735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.474740:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.474744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763506.474751:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763506.474772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.474784:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020daea00 x1876702067666560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.474847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067666560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.474859:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.474863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.474867:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763506.474906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763506.474935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.474950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020dad180 x1876702067666688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.474973:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067666688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.474984:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.474988:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.474992:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763506.474998:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763506.475018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.475030:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020dae300 x1876702067666816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.475044:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067666816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.475053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.475057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.475061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067666816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763506.475068:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763506.475088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.475101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020daf800 x1876702067666944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.475114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067666944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.475132:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.475135:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.475139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067666944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763506.475145:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763506.475165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763506.475177:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763506/real 1789763506] req@ffff936020dadf80 x1876702067667072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763506.475190:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067667072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763506.475199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763506.475202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763506.475206:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667072:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763507.242053:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763508.266020:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763509.290195:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763510.314075:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763511.337971:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763511.593946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.593959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763511.593966:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763511.593985:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594003:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763511.594073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067667200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763511.594123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.594129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763511.594131:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763511.594137:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594142:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594147:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594161:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763511.594183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067667328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763511.594201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.594208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763511.594214:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763511.594215:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067667200:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763511.594228:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594237:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594242:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594251:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763511.594269:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067667456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763511.594282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.594287:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763511.594290:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763511.594297:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594303:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594307:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594313:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763511.594328:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067667584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763511.594366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.594373:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763511.594376:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763511.594381:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763511.594387:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594396:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594413:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594428:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763511.594451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067667712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763511.594466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763511.594471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763511.594475:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763511.594480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020dad500 x1876702067667200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763511.594488:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594500:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594513:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594522:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763511.594535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067667200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763511.594546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067667840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763511.594567:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763511.594579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763511.594581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763511.594593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763511.594601:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763511.594607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067667200:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763511.594614:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 979 00000100:00080000:0.0:1789763511.594623:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763511.594628:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763511.594634:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763511.594654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067667968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763511.594660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763511.594748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.594771:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020daf100 x1876702067667328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.594789:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067667328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.594870:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.594896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.594906:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763511.594925:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763511.594963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.594981:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020dac700 x1876702067667456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.595010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067667456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.595022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.595026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.595031:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763511.595038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763511.595067:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.595090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020dad880 x1876702067667584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.595115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067667584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.595134:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.595141:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.595149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763511.595161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763511.595197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.595214:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020daf480 x1876702067667712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.595229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067667712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.595240:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.595244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.595248:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067667712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763511.595255:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763511.595277:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.595290:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020dac380 x1876702067667840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.595304:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067667840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.595314:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.595318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.595321:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763511.595377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763511.595407:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763511.595422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763511/real 1789763511] req@ffff936020dae680 x1876702067667968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763511.595436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067667968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763511.595446:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763511.595449:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763511.595453:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067667968:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763512.361966:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763513.386025:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763514.410053:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763515.434109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763516.458129:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763516.713940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.713954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763516.713962:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.713977:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.713993:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714004:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763516.714065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067668096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763516.714107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763516.714116:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714121:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714126:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714144:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763516.714160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067668224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763516.714175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763516.714183:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714188:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714191:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714196:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714201:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763516.714220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067668352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763516.714230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067668096:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763516.714239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763516.714253:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714260:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714266:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714274:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714282:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763516.714327:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067668480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763516.714341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763516.714358:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714363:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714368:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714373:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714381:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763516.714401:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067668608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763516.714403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763516.714428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714434:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763516.714436:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714445:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714452:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714458:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714464:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763516.714486:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067668736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763516.714502:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061dd180 x1876702067668096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763516.714511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763516.714534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763516.714550:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763516.714567:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 984 00000100:00080000:2.0:1789763516.714581:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763516.714590:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763516.714600:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763516.714615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067668096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763516.714617:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067668864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763516.714750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.714758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.714767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067668096:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763516.714779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763516.714880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.714899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061dce00 x1876702067668224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.714916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067668224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.714926:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.714930:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.714934:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763516.714966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763516.714989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.715001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061dc000 x1876702067668352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.715015:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067668352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.715024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.715027:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.715031:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763516.715039:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763516.715059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.715070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061de680 x1876702067668480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.715083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067668480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.715093:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.715096:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.715100:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763516.715106:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763516.715125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.715136:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061dfb80 x1876702067668608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.715148:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067668608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.715156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.715159:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.715162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067668608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763516.715168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067668736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763516.715187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.715198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061df800 x1876702067668736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.715212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067668736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.715221:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.715224:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.715228:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067668736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763516.715234:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067668864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763516.715253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763516.715264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763516/real 1789763516] req@ffff9360061dca80 x1876702067668864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763516.715278:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067668864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763516.715286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763516.715290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763516.715293:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067668864:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763517.481994:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763518.506024:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763519.530103:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763520.554089:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763521.578130:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763521.834017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763521.834040:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834053:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834067:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834077:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763521.834129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067668992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763521.834180:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834185:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834189:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834193:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834205:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763521.834219:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067669120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763521.834250:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834255:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834259:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834295:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834304:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763521.834318:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067668992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763521.834326:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067669248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763521.834353:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834358:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834364:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834369:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834375:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763521.834391:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067669376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763521.834410:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834414:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834419:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834427:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763521.834438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834439:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763521.834462:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067669504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763521.834482:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834486:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834490:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834495:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834501:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763521.834534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa6d80 x1876702067668992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763521.834535:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067669632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763521.834562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763521.834566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763521.834568:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763521.834579:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 989 00000100:00080000:3.0:1789763521.834589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763521.834598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763521.834604:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763521.834615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067668992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763521.834621:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067669760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763521.834670:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.834679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.834687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067668992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763521.834697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763521.834735:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.834753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa4000 x1876702067669120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.834769:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067669120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.834782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.834789:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.834871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763521.834921:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763521.834954:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.834970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa6300 x1876702067669248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.834986:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067669248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.834998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.835002:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.835006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763521.835014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763521.835036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.835048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa6680 x1876702067669376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.835062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067669376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.835072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.835075:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.835079:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763521.835085:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763521.835104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.835116:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa5880 x1876702067669504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.835129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067669504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.835139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.835143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.835146:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067669504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763521.835152:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067669632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763521.835172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.835184:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa7480 x1876702067669632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.835198:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067669632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.835208:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.835211:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.835215:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067669632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763521.835221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067669760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763521.835239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763521.835251:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763521/real 1789763521] req@ffff936010aa5c00 x1876702067669760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763521.835295:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067669760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763521.835306:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763521.835310:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763521.835313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067669760:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763522.601961:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763523.625987:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763524.650121:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763525.674062:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763526.697954:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763526.953990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763526.954008:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954020:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954032:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954040:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763526.954083:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067669888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763526.954119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763526.954126:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954130:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954134:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954138:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763526.954150:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763526.954162:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067670016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763526.954175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763526.954182:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954186:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954189:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954192:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763526.954197:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763526.954209:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067670144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763526.954220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067669888:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763526.954245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954257:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763526.954260:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954317:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954324:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954330:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763526.954339:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763526.954358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763526.954366:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067670272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763526.954383:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954388:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763526.954392:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954397:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954402:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954409:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763526.954417:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763526.954427:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061dd500 x1876702067669888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763526.954437:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067670400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763526.954474:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763526.954479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763526.954483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067669888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763526.954484:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954498:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954506:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763526.954520:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763526.954527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789763526.954528:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763526.954544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763526.954556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067670528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763526.954557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067669888:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763526.954583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763526.954584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670016:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763526.954619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763526.954623:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763526.954633:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 994 00000100:00080000:2.0:1789763526.954641:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763526.954648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763526.954656:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763526.954668:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763526.954670:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061df480 x1876702067670016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.954690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067670016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763526.954699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067670656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763526.954704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.954736:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.954749:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763526.954788:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763526.954814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763526.954902:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061df100 x1876702067670144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.954923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067670144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763526.954933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.954936:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.954940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763526.954947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763526.954973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763526.954988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061dd880 x1876702067670272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.955009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067670272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763526.955023:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.955028:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.955034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763526.955045:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763526.955079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763526.955096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061ddc00 x1876702067670400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.955109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067670400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763526.955118:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.955122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.955125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763526.955131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067670528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763526.955161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763526.955180:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061de300 x1876702067670528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.955203:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067670528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763526.955218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.955247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.955251:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067670528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763526.955259:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067670656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763526.955308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763526.955323:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763526/real 1789763526] req@ffff9360061ddf80 x1876702067670656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763526.955338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067670656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763526.955348:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763526.955352:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763526.955356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067670656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763527.722040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763528.746064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763529.769965:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763530.794104:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763531.818014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763532.073988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763532.074023:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074040:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074058:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074069:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763532.074126:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067670784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763532.074210:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074220:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074229:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074238:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074255:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763532.074262:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067670784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763532.074287:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067670912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763532.074324:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074333:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074341:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074349:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763532.074384:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067671040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763532.074416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670912:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763532.074417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763532.074427:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074442:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074450:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074459:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074472:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763532.074501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763532.074503:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067671168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763532.074558:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074567:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00100000:1.0:1789763532.074577:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671168:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763532.074577:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074594:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074618:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763532.074647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067671296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074694:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763532.074700:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074708:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074716:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763532.074721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671296:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763532.074732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074747:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763532.074777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067671424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763532.074870:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763532.074880:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763532.074883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067671424:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763532.074887:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763532.074915:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 999 00000100:00080000:3.0:1789763532.074926:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763532.074935:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763532.074955:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763532.074983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067671552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763532.075098:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067671552:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763532.841920:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763533.866058:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763534.890060:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763535.146213:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763535.146285:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763535.146312:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763535.146368:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763535.146390:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763535.146423:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763535.146441:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa4e00 x1876702067670912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763535.146448:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763535.146474:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763535.146511:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067670912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763535.146514:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:1.0:1789763535.146545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.146556:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.146565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067670912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763535.146579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa4700 x1876702067671040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.146604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067671040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.146620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.146627:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.146633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763535.146646:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa5f80 x1876702067670784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.146711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067670784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.146727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.146735:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.146742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067670784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763535.146789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa5880 x1876702067671552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.146866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067671552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.146884:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.146891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.146899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067671552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763535.146912:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa7480 x1876702067671424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.146937:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067671424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.146953:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.146959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.146965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067671424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763535.146978:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa5c00 x1876702067671296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.147003:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067671296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.147017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.147023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.147028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763535.147040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763532/real 1789763535] req@ffff936010aa6a00 x1876702067671168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763535.147064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067671168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763535.147080:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763535.147102:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763535.147110:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671168:192.168.203.137@tcp:8:kworker.0 00010000:00010000:2.0:1789763535.913992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763536.938055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763537.193943:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.193957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763537.193980:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763537.193995:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194009:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194019:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763537.194062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067671680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763537.194098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.194102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763537.194105:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763537.194110:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194121:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194125:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194136:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763537.194180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067671808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763537.194218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.194226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763537.194231:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763537.194243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067671680:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763537.194247:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194263:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194275:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194285:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763537.194308:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067671936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763537.194334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.194338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763537.194342:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763537.194347:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194353:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194358:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194387:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763537.194395:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763537.194410:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067672064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763537.194430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.194436:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763537.194454:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763537.194464:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00100000:1.0:1789763537.194470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa6680 x1876702067671680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763537.194475:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194493:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194509:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763537.194527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067671680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763537.194539:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067672192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763537.194563:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763537.194572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763537.194580:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763537.194585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763537.194594:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763537.194598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067671680:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763537.194609:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194620:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763537.194625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671808:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763537.194635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194650:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763537.194677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067672320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763537.194695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763537.194713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763537.194721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763537.194724:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763537.194732:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1005 00000100:00080000:3.0:1789763537.194739:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763537.194746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763537.194757:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763537.194759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa6300 x1876702067671808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763537.194786:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067672448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763537.194880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067671808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.194902:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.194909:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.194917:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763537.194959:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763537.195003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763537.195026:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa4000 x1876702067671936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763537.195050:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067671936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.195065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.195071:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.195077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067671936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763537.195088:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763537.195127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763537.195181:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa6d80 x1876702067672064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763537.195208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067672064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.195226:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.195233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.195239:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763537.195251:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763537.195292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763537.195319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa7100 x1876702067672192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763537.195342:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067672192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.195358:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.195364:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.195370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763537.195383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067672320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763537.195432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763537.195454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa5500 x1876702067672320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763537.195488:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067672320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.195503:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.195509:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.195514:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067672320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763537.195526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067672448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763537.195565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763537.195588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763537/real 1789763537] req@ffff936010aa4a80 x1876702067672448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763537.195613:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067672448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763537.195629:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763537.195634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763537.195640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067672448:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763537.962029:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763538.985972:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763540.009928:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763541.034021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763542.058001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763542.313956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763542.313970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763542.313975:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763542.313989:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314004:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763542.314013:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763542.314063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067672576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763542.314140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763542.314146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763542.314149:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763542.314155:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314160:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763542.314165:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314178:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763542.314197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067672704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763542.314213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763542.314218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763542.314225:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763542.314228:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067672576:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763542.314237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314252:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763542.314260:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314271:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763542.314297:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067672832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763542.314317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763542.314323:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763542.314327:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763542.314335:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314344:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763542.314357:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763542.314359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314390:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763542.314410:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067672960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763542.314424:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763542.314427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763542.314429:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763542.314433:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314439:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763542.314440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa7800 x1876702067672576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763542.314447:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314473:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763542.314492:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067672576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763542.314497:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067673088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763542.314556:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763542.314557:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763542.314570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763542.314576:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763542.314577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763542.314587:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00100000:1.0:1789763542.314593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067672576:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763542.314597:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763542.314612:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763542.314641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067673216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763542.314648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672704:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763542.314699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763542.314708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763542.314708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763542.314714:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763542.314722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1010 00000100:00080000:3.0:1789763542.314752:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763542.314762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa7b80 x1876702067672704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763542.314763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763542.314779:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763542.314821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067673344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763542.314842:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067672704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.314878:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.314885:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.314892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763542.314905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763542.314937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763542.314953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa5180 x1876702067672832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763542.314968:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067672832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.314978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.314982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.314986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763542.314993:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763542.315014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763542.315027:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa4380 x1876702067672960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763542.315041:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067672960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.315051:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.315054:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.315058:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067672960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763542.315064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763542.315084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763542.315096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa4a80 x1876702067673088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763542.315135:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067673088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.315148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.315152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.315156:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763542.315182:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067673216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763542.315211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763542.315225:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa5500 x1876702067673216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763542.315247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067673216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.315266:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.315272:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.315278:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067673216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763542.315291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067673344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763542.315330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763542.315353:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763542/real 1789763542] req@ffff936010aa7100 x1876702067673344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763542.315387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067673344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763542.315395:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763542.315398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763542.315401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067673344:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763543.082023:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763544.106047:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763545.129950:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763546.154063:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763547.178021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763547.433940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.433952:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763547.433957:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.433971:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.433986:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.433995:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763547.434051:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067673472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763547.434099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.434107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763547.434109:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.434114:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434118:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434124:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763547.434137:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763547.434152:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067673600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763547.434158:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067673472:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763547.434170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.434175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763547.434176:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.434181:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434184:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434187:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763547.434192:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763547.434205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067673728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763547.434215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.434219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763547.434221:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.434225:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434228:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434231:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763547.434238:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763547.434250:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067673856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763547.434255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763547.434269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.434273:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763547.434276:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.434280:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434285:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434288:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763547.434292:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763547.434304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067673984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763547.434313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763547.434320:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061dc380 x1876702067673472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763547.434320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763547.434326:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763547.434337:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434347:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434353:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763547.434359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067673472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763547.434362:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763547.434380:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.434389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763547.434391:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067674112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763547.434395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067673472:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763547.434417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763547.434423:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763547.434427:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763547.434433:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673600:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763547.434436:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1015 00000100:00080000:2.0:1789763547.434447:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763547.434452:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763547.434459:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763547.434466:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763547.434478:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067674240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763547.434486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061dc700 x1876702067673600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.434508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067673600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.434518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.434521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.434525:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763547.434531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763547.434552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763547.434562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061ddf80 x1876702067673728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.434584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067673728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.434595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.434600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.434603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763547.434618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763547.434644:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763547.434655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061de300 x1876702067673856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.434670:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067673856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.434678:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.434681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.434683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763547.434688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763547.434707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763547.434724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061ddc00 x1876702067673984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.434768:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067673984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.434786:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.434795:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.434803:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067673984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763547.434890:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067674112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763547.434939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763547.434963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061dd880 x1876702067674112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.434992:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067674112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.435008:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.435012:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.435017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067674112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763547.435070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067674240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763547.435126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763547.435144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763547/real 1789763547] req@ffff9360061df100 x1876702067674240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763547.435162:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067674240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763547.435171:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763547.435175:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763547.435179:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067674240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:2.0:1789763548.201992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763549.226098:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763550.250136:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763551.274104:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763552.298157:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763552.553971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.553987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763552.553993:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554011:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554029:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554079:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763552.554143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067674368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763552.554184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.554189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763552.554198:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554205:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554211:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554217:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554231:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763552.554248:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067674496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763552.554262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.554267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763552.554269:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554275:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554281:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763552.554285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067674368:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763552.554292:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554304:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763552.554319:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067674624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763552.554333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.554338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763552.554340:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554346:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554352:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554356:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554361:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763552.554377:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067674752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763552.554386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763552.554403:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.554409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763552.554412:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554419:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554423:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554427:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554433:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763552.554451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067674880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763552.554458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa6d80 x1876702067674368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763552.554482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763552.554492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763552.554495:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554507:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00100000:1.0:1789763552.554508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067674368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763552.554512:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554524:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554539:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763552.554545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763552.554555:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067675008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763552.554557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.554573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067674368:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763552.554592:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763552.554598:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674496:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763552.554601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763552.554609:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763552.554618:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1020 00000100:00080000:3.0:1789763552.554624:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763552.554630:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763552.554637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763552.554644:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763552.554663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067675136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763552.554680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa4000 x1876702067674496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.554713:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067674496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.554732:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.554738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.554745:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763552.554787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763552.554885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763552.554911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa6300 x1876702067674624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.554937:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067674624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.554954:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.554961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.554968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763552.554982:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763552.555026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763552.555078:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa6680 x1876702067674752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.555103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067674752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.555120:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.555126:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.555133:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763552.555147:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763552.555182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763552.555198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa6a00 x1876702067674880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.555213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067674880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.555223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.555227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.555231:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067674880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763552.555237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763552.555258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763552.555271:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa5c00 x1876702067675008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.555285:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067675008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.555295:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.555298:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.555302:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763552.555308:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763552.555328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763552.555340:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763552/real 1789763552] req@ffff936010aa7480 x1876702067675136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763552.555354:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067675136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763552.555363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763552.555366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763552.555370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763553.322154:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763554.345984:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763555.370041:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763556.394149:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763557.417890:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763557.673896:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.673905:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763557.673909:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.673918:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.673927:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.673933:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763557.673985:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067675264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763557.674030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.674033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763557.674034:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674045:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674054:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763557.674065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067675392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763557.674074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763557.674075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067675264:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763557.674077:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763557.674081:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674087:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674089:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674091:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674094:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763557.674104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067675520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763557.674111:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.674113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763557.674115:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674117:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674121:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674123:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674126:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763557.674133:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067675648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763557.674139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.674141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763557.674143:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674145:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674147:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674150:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763557.674155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674156:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763557.674166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067675776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763557.674173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.674175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763557.674177:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674179:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674183:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763557.674195:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067675904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763557.674202:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef6d80 x1876702067675264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763557.674205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763557.674212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763557.674215:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763557.674222:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1025 00000100:00080000:1.0:1789763557.674226:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763557.674228:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763557.674231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763557.674239:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067676032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763557.674239:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067675264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674271:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674275:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674279:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067675264:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763557.674300:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763557.674320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef7480 x1876702067675392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674354:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067675392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674367:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763557.674377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763557.674393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674401:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef7b80 x1876702067675520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067675520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674416:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674419:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763557.674422:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763557.674433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef7800 x1876702067675648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674447:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067675648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674458:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763557.674463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763557.674473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674479:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef4380 x1876702067675776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067675776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067675776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763557.674497:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763557.674508:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674514:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef4000 x1876702067675904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067675904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674526:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674528:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674530:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067675904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763557.674534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763557.674545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763557.674551:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763557/real 1789763557] req@ffff936010ef5c00 x1876702067676032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763557.674558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067676032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763557.674562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763557.674564:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763557.674566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676032:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763558.442056:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763559.466022:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763560.490012:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763561.514091:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763562.538127:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763562.794024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763562.794039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763562.794045:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763562.794059:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794074:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763562.794084:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763562.794134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067676160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763562.794172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763562.794177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763562.794180:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763562.794185:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794189:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763562.794194:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763562.794208:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763562.794224:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067676288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763562.794243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763562.794246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067676160:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763562.794250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763562.794259:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763562.794274:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794281:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763562.794289:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763562.794300:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763562.794328:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067676416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763562.794350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763562.794357:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763562.794362:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763562.794369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763562.794377:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794387:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763562.794395:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763562.794404:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763562.794425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067676544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763562.794440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa5880 x1876702067676160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763562.794444:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763562.794466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763562.794476:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763562.794490:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00100000:2.0:1789763562.794492:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067676160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763562.794507:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763562.794514:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763562.794524:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789763562.794526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.794540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067676160:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763562.794546:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763562.794581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763562.794592:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067676672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763562.794622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763562.794635:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763562.794639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763562.794647:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763562.794660:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794671:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763562.794673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa5f80 x1876702067676288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763562.794688:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763562.794705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763562.794724:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067676288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763562.794733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067676800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763562.794765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763562.794771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763562.794777:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763562.794782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763562.794789:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676288:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763562.794791:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763562.794869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676416:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763562.794877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1030 00000100:00080000:3.0:1789763562.794887:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763562.794894:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763562.794903:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763562.794918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763562.794927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067676928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763562.794945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa4700 x1876702067676416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763562.795012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067676416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763562.795026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763562.795032:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.795038:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763562.795049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763562.795081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763562.795096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa4e00 x1876702067676544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763562.795121:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067676544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763562.795131:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763562.795135:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.795139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763562.795161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763562.795185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763562.795198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa7480 x1876702067676672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763562.795213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067676672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763562.795223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763562.795227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.795231:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067676672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763562.795238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763562.795273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763562.795295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa5c00 x1876702067676800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763562.795323:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067676800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763562.795353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763562.795358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.795363:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763562.795371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763562.795400:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763562.795415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763562/real 1789763562] req@ffff936010aa6a00 x1876702067676928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763562.795429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067676928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763562.795439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763562.795443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763562.795447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067676928:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789763563.562021:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763564.586024:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763565.610071:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763566.634062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763567.658077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763567.913919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763567.913958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763567.913964:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.913981:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914000:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763567.914014:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763567.914074:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067677056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763567.914114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763567.914120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763567.914123:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914128:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914134:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763567.914139:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763567.914153:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763567.914181:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067677184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763567.914197:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067677056:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763567.914234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763567.914243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763567.914248:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914261:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914274:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763567.914281:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763567.914293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763567.914294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763567.914325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067677312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763567.914341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763567.914346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763567.914349:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914356:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914364:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763567.914365:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa6680 x1876702067677056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763567.914377:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763567.914392:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763567.914417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067677056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763567.914420:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6d80 x1876702067677440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763567.914471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763567.914472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763567.914483:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.914494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067677056:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763567.914495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763567.914505:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914515:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914520:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763567.914525:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763567.914532:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763567.914540:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763567.914561:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067677568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763567.914579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763567.914584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763567.914587:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914592:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:3.0:1789763567.914598:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763567.914602:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763567.914611:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763567.914621:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763567.914637:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa6300 x1876702067677184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763567.914645:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067677696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763567.914687:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763567.914692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067677184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763567.914694:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763567.914701:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763567.914719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1035 00000100:00080000:2.0:1789763567.914720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.914734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763567.914734:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763567.914746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677184:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763567.914755:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763567.914763:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677312:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763567.914774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763567.914846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763567.914859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067677824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763567.914873:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa4000 x1876702067677312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763567.914903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067677312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763567.914917:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.914952:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.914957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763567.914966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763567.914992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763567.915008:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa6d80 x1876702067677440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763567.915023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6d80 x1876702067677440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763567.915033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.915037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.915041:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763567.915066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763567.915088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763567.915101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa7100 x1876702067677568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763567.915115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067677568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763567.915125:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.915128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.915132:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067677568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763567.915138:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067677696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763567.915158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763567.915171:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa5500 x1876702067677696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763567.915184:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067677696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763567.915195:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.915199:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.915202:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067677696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763567.915208:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067677824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763567.915229:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763567.915241:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763567/real 1789763567] req@ffff936010aa4a80 x1876702067677824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763567.915254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067677824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763567.915263:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763567.915267:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763567.915271:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067677824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763568.682061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763569.706048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763570.730050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763571.754114:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763572.778048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763573.033988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034004:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763573.034010:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034024:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034038:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763573.034047:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763573.034098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067677952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763573.034145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763573.034154:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034159:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034163:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763573.034168:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763573.034214:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067678080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763573.034231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763573.034232:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067677952:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763573.034246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763573.034251:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034260:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763573.034264:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034270:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763573.034287:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067678208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763573.034299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763573.034306:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034309:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034348:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763573.034355:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034361:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763573.034374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763573.034385:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067678336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763573.034409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034415:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763573.034420:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034429:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034438:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763573.034450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa4380 x1876702067677952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763573.034451:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034471:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763573.034489:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067677952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.034510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.034517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763573.034522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067678464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763573.034528:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067677952:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763573.034554:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763573.034581:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034588:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00100000:2.0:1789763573.034593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678080:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763573.034597:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763573.034608:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763573.034652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067678592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763573.034676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763573.034676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763573.034686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763573.034689:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763573.034697:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1040 00000100:00080000:3.0:1789763573.034705:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763573.034710:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa5180 x1876702067678080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763573.034716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763573.034731:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763573.034742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067678080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.034756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763573.034759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067678720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763573.034766:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.034778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763573.034839:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763573.034933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763573.034973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa7b80 x1876702067678208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763573.035022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067678208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.035033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.035038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.035042:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763573.035051:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763573.035083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763573.035098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa7800 x1876702067678336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763573.035114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067678336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.035124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.035128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.035132:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763573.035139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763573.035160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763573.035172:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa4a80 x1876702067678464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763573.035186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067678464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.035196:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.035199:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.035203:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067678464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763573.035246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067678592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763573.035268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763573.035281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa5500 x1876702067678592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763573.035303:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067678592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.035378:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.035387:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.035395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067678592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763573.035409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067678720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763573.035442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763573.035458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763573/real 1789763573] req@ffff936010aa7100 x1876702067678720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763573.035477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067678720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763573.035493:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763573.035497:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763573.035501:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067678720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763573.802131:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763574.826052:0:255:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/2 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1789763574.826088:0:255:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 2/4 locks to regular blp list, 2 blwis in pool 00010000:00010000:0.0:1789763574.826143:0:6358:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 2 locks of 2 total in blp. (1 blwis in pool) 00010000:00010000:0.0:1789763574.826176:0:6358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:3.0:1789763574.826190:0:6357:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 2 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:1.0:1789763574.826194:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763574.826213:0:6357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789763574.826255:0:6358:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482800/0xb758ac0edef9cbc7 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xdf5a7ac59f177bdf expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000080:00200000:3.0:1789763574.826265:0:6357:0:(vvp_object.c:117:vvp_conf_set()) [0x240000402:0x1a:0x0]: losing layout lock 00010000:00010000:3.0:1789763574.826320:0:6357:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 2/0,0 mode: PR/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789763574.826332:0:6358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 1/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00010000:00010000:0.0:1789763574.826379:0:6358:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482800/0xb758ac0edef9cbc7 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bdf expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:0.0:1789763574.826389:0:6358:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:3.0:1789763574.826430:0:6357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 1/0,0 mode: --/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789763574.826432:0:6358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001483e00/0xb758ac0edef9cbb9 lrc: 0/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x1/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bca expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 0 00000100:00080000:1.0:1789763574.826485:0:2395:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff936020daca80 x1876702067678848/t0(0) o103->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1789763574.826509:0:6357:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 1/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:0.0:1789763574.826527:0:6358:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff936020592000 lock: ffff936001482800/0xb758ac0edef9cbc7 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177bdf expref: -99 pid: 16333 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1789763574.826530:0:6357:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 2 locks packed 00010000:00010000:0.0:1789763574.826562:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763574.826567:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763574.826573:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763574.826584:0:6357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cbe00/0xb758ac0edef9cbf1 lrc: 0/0,0 mode: --/PR res: [0x240000402:0x1a:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0xdf5a7ac59f177c6b expref: -99 pid: 16340 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00000100:00080000:2.0:1789763574.826651:0:2398:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff936010aa6d80 x1876702067678976/t0(0) o103->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1789763574.826654:0:6357:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0001-mdc-ffff936020592000 lock: ffff9360421cac00/0xb758ac0edef9cbea lrc: 0/0,0 mode: --/PR res: [0x240000402:0x19:0x0].0x0 bits 0x10/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xdf5a7ac59f177c4f expref: -99 pid: 16340 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789763574.826706:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763574.826711:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763574.826715:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763575.850088:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763576.874078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763577.898068:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763578.153968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.153982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763578.153987:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154002:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154015:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763578.154025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763578.154073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067679104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763578.154103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.154107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763578.154110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154114:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154118:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763578.154121:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763578.154133:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763578.154146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067679232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763578.154168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.154172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763578.154175:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763578.154180:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067679104:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763578.154184:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154195:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763578.154202:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763578.154211:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763578.154230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067679360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763578.154249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.154256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763578.154260:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154269:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154277:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763578.154281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763578.154293:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763578.154305:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763578.154343:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067679488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763578.154355:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa4000 x1876702067679104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763578.154363:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.154377:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763578.154382:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154391:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154397:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763578.154402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067679104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763578.154408:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763578.154420:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763578.154428:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.154438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763578.154441:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067679616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763578.154450:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067679104:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763578.154469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763578.154473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763578.154475:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154481:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154484:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763578.154490:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763578.154495:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679232:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763578.154499:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763578.154519:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067679744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763578.154531:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763578.154539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763578.154541:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763578.154546:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763578.154553:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1045 00000100:00080000:3.0:1789763578.154559:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763578.154584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763578.154586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa6300 x1876702067679232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763578.154591:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763578.154618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067679872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763578.154623:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067679232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.154665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.154678:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.154684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763578.154693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763578.154746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763578.154767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa6680 x1876702067679360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763578.154788:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067679360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.154803:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.154809:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.154813:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763578.154821:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763578.154913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763578.154928:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa6a00 x1876702067679488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763578.154941:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067679488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.154951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.154954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.154958:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763578.154964:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763578.154990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763578.155008:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa5c00 x1876702067679616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763578.155030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067679616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.155045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.155050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.155056:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067679616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763578.155089:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067679744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763578.155115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763578.155128:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa7480 x1876702067679744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763578.155141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067679744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.155151:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.155154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.155157:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067679744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763578.155163:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067679872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763578.155180:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763578.155191:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763578/real 1789763578] req@ffff936010aa4e00 x1876702067679872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763578.155203:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067679872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763578.155211:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763578.155214:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763578.155217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067679872:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763578.922078:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763579.946086:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763580.970009:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763581.994066:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763583.018018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763583.273953:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763583.273968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763583.273974:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.273988:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274002:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763583.274012:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763583.274060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067680000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763583.274102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763583.274108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763583.274110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.274115:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274120:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763583.274124:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763583.274154:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763583.274170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067680128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763583.274184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763583.274189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763583.274192:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.274197:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274201:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763583.274208:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067680000:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763583.274212:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763583.274244:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763583.274263:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067680256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763583.274279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763583.274286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763583.274290:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.274298:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274307:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763583.274316:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763583.274341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763583.274349:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763583.274379:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067680384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763583.274399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763583.274406:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa4700 x1876702067680000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763583.274408:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763583.274413:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.274427:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274438:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763583.274446:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763583.274454:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067680000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763583.274455:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763583.274481:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067680512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763583.274490:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763583.274500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763583.274502:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763583.274510:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763583.274514:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763583.274519:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067680000:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763583.274522:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274531:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763583.274539:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763583.274545:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763583.274551:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763583.274561:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067680640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763583.274576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763583.274580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763583.274582:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763583.274588:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1051 00000100:00080000:3.0:1789763583.274606:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763583.274611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763583.274620:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763583.274632:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763583.274641:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa5f80 x1876702067680128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763583.274680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067680768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763583.274683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067680128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.274743:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.274760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.274767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763583.274805:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763583.274970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763583.274991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa5880 x1876702067680256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763583.275009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067680256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.275023:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.275030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.275038:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763583.275053:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763583.275110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763583.275163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa4e00 x1876702067680384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763583.275192:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067680384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.275207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.275213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.275218:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763583.275226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763583.275261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763583.275279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa7480 x1876702067680512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763583.275293:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067680512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.275302:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.275305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.275308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067680512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763583.275315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067680640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763583.275336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763583.275352:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa5c00 x1876702067680640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763583.275372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067680640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.275382:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.275386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.275392:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067680640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763583.275401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067680768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763583.275426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763583.275444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763583/real 1789763583] req@ffff936010aa6a00 x1876702067680768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763583.275463:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067680768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763583.275473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763583.275478:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763583.275482:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067680768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763584.042031:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763585.066038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763586.089994:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763587.114034:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763588.138028:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763588.393926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763588.393942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763588.393948:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763588.393963:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.393978:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.393988:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763588.394036:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067680896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763588.394080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763588.394087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763588.394091:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763588.394099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.394106:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.394113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763588.394129:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763588.394155:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067681024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763588.394179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763588.394184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763588.394187:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763588.394193:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.394201:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763588.394210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067680896:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763588.394212:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763588.394229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763588.394251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067681152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763588.394267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763588.394272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763588.394274:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763588.394279:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.394285:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.394290:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763588.394297:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763588.394315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763588.394316:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067681280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763588.394349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763588.394354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763588.394357:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763588.394364:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.394369:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.394378:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763588.394380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061df480 x1876702067680896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763588.394385:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763588.394414:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067681408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763588.394423:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067680896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763588.394463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763588.394470:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763588.394474:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763588.394483:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763588.394490:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763588.394499:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00100000:1.0:1789763588.394511:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067680896:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763588.394514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.394530:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763588.394539:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763588.394541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763588.394562:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067681536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763588.394587:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763588.394593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763588.394595:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763588.394617:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763588.394620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061dd500 x1876702067681024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763588.394628:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1056 00000100:00080000:2.0:1789763588.394639:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763588.394648:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763588.394655:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763588.394657:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067681024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.394673:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763588.394675:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067681664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763588.394686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.394702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763588.394745:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763588.394770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763588.394866:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061dca80 x1876702067681152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763588.394882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067681152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.394893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763588.394897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.394901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763588.394909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763588.394936:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763588.394949:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061df800 x1876702067681280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763588.394963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067681280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.394972:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763588.394976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.394979:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763588.394986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763588.395006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763588.395025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061dfb80 x1876702067681408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763588.395043:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067681408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.395053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763588.395057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.395061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763588.395068:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067681536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763588.395094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763588.395111:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061de680 x1876702067681536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763588.395126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067681536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.395136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763588.395139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.395142:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067681536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763588.395149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067681664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763588.395168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763588.395181:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763588/real 1789763588] req@ffff9360061dc000 x1876702067681664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763588.395194:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067681664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763588.395212:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763588.395216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763588.395219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067681664:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763589.161924:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763590.186043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763591.210037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763592.234065:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763593.257942:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763593.513881:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.513893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763593.513897:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.513909:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.513922:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.513929:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763593.513980:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067681792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763593.514015:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514018:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514021:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514024:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763593.514035:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763593.514047:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067681920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763593.514079:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514082:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514084:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514086:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763593.514089:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763593.514099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067682048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514105:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763593.514109:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514114:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514117:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514121:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763593.514126:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763593.514137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067682176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514154:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763593.514158:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514169:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514173:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514177:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763593.514181:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067681792:192.168.203.137@tcp:250:kworker.0 00000008:00100000:2.0:1789763593.514183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763593.514200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067682304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763593.514211:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514214:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514217:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514222:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763593.514227:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763593.514242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067682432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763593.514255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763593.514259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763593.514261:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763593.514266:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1061 00000100:00080000:2.0:1789763593.514272:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763593.514275:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763593.514279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763593.514289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763593.514292:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067682560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763593.514404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061dce00 x1876702067681792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.514453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067681792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.514495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.514515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.514523:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067681792:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763593.514541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763593.514588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.514615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061dd180 x1876702067681920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.514641:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067681920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.514668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.514677:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.514684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067681920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763593.514715:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763593.514888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.514924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061dc000 x1876702067682048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.514950:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067682048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.514968:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.514975:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.514982:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763593.515032:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763593.515085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.515107:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061de680 x1876702067682176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.515131:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067682176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.515148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.515154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.515160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763593.515172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763593.515210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.515232:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061dfb80 x1876702067682304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.515255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067682304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.515269:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.515275:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.515281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763593.515293:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067682432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763593.515327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.515347:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061df800 x1876702067682432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.515369:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067682432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.515383:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.515389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.515395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067682432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763593.515407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067682560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763593.515442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763593.515464:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763593/real 1789763593] req@ffff9360061dca80 x1876702067682560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763593.515499:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067682560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763593.515515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763593.515521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763593.515527:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067682560:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763594.282051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763595.306032:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763596.330046:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763597.354008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763598.378007:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763598.633944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.633958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763598.633964:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763598.633978:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.633992:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763598.634053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067682688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763598.634092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763598.634100:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763598.634105:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.634109:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763598.634142:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067682816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763598.634157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763598.634164:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763598.634173:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00100000:1.0:1789763598.634178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067682688:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763598.634179:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634192:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634202:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763598.634221:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067682944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763598.634235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763598.634271:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763598.634276:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.634282:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634286:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763598.634321:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067683072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763598.634335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763598.634342:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763598.634347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682816:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763598.634353:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.634359:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634363:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634371:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763598.634397:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067683200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763598.634413:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682944:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763598.634420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763598.634434:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763598.634446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683072:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763598.634447:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.634464:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634477:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634486:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763598.634492:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763598.634514:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067683328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763598.634533:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763598.634537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763598.634540:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763598.634547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067683328:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763598.634552:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1066 00000100:00080000:3.0:1789763598.634563:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763598.634568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763598.634574:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763598.634591:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067683456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763598.634647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067683456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763599.402029:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763600.425998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763601.450026:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763601.706214:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763601.706302:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763601.706330:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706398:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706416:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706434:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706452:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706466:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763601.706481:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763601.706510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa4a80 x1876702067683328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.706562:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067683328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.706600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.706609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.706617:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067683328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763601.706633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa5500 x1876702067683200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.706660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067683200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.706707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.706715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.706723:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763601.706736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa4000 x1876702067682944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.706783:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067682944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.706849:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.706861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.706870:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763601.706887:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa7100 x1876702067683072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.706916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067683072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.706936:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.706943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.706950:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763601.706969:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa6680 x1876702067682688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.707005:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067682688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.707023:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.707032:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.707039:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067682688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763601.707052:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa6300 x1876702067682816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.707075:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067682816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.707091:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.707098:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.707105:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067682816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763601.707164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763598/real 1789763601] req@ffff936010aa7800 x1876702067683456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763601.707191:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067683456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763601.707206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763601.707214:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763601.707220:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067683456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763602.474032:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763603.498017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763603.753997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763603.754022:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763603.754040:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754056:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754070:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763603.754121:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067683584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763603.754159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763603.754166:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763603.754171:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754177:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754181:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754196:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763603.754220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067683712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763603.754241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763603.754253:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763603.754264:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067683584:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763603.754271:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754283:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754291:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763603.754330:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067683840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763603.754354:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763603.754364:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763603.754371:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754380:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754390:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763603.754399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754408:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763603.754440:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067683968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763603.754462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763603.754474:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763603.754485:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061dd500 x1876702067683584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763603.754488:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754501:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754515:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754531:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763603.754546:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067683584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763603.754558:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067684096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763603.754589:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763603.754595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763603.754606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763603.754608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763603.754616:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763603.754625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067683584:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763603.754634:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754646:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763603.754652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683712:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763603.754690:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763603.754735:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067684224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763603.754753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763603.754771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763603.754780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763603.754785:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763603.754795:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061df480 x1876702067683712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763603.754840:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1071 00000100:00080000:2.0:1789763603.754850:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763603.754859:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763603.754870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763603.754899:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067684352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763603.754925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067683712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.754949:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.754959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.754967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763603.755015:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763603.755063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763603.755089:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061df100 x1876702067683840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763603.755114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067683840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.755131:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.755138:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.755145:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763603.755158:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763603.755201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763603.755237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061dd880 x1876702067683968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763603.755263:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067683968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.755282:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.755288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.755296:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067683968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763603.755309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763603.755352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763603.755378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061ddc00 x1876702067684096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763603.755405:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067684096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.755422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.755428:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.755434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763603.755447:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067684224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763603.755488:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763603.755513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061de300 x1876702067684224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763603.755536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067684224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.755551:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.755557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.755562:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067684224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763603.755573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067684352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763603.755612:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763603.755649:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763603/real 1789763603] req@ffff9360061ddf80 x1876702067684352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763603.755700:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067684352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763603.755731:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763603.755737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763603.755744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067684352:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763604.522019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763605.546035:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763606.569991:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763607.594014:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763608.617998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763608.873964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.873979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763608.873985:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763608.873999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874013:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874022:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763608.874071:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067684480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763608.874117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.874125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763608.874129:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763608.874137:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874151:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763608.874168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763608.874193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067684608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763608.874214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.874221:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763608.874225:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763608.874233:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874248:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763608.874258:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763608.874282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067684736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763608.874294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067684480:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763608.874322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.874330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763608.874334:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763608.874344:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874354:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874361:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763608.874370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763608.874396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763608.874397:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067684864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763608.874417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.874421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763608.874424:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763608.874430:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874435:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874439:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763608.874447:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763608.874466:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef4700 x1876702067684480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763608.874469:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067684992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763608.874537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763608.874546:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067684480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763608.874550:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763608.874560:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763608.874575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763608.874579:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:2.0:1789763608.874586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.874597:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067684480:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763608.874598:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874651:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763608.874654:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684608:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789763608.874670:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763608.874700:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067685120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763608.874714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763608.874730:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763608.874736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763608.874739:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763608.874747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef5180 x1876702067684608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763608.874751:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1076 00000100:00080000:1.0:1789763608.874765:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763608.874776:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763608.874783:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067684608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789763608.874788:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763608.874843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.874854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.874863:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763608.874907:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067685248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763608.874916:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763608.874989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763608.875009:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef4a80 x1876702067684736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763608.875026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067684736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763608.875037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.875041:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.875045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763608.875053:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763608.875075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763608.875087:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef5880 x1876702067684864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763608.875101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067684864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763608.875111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.875115:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.875118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763608.875125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763608.875145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763608.875157:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef5c00 x1876702067684992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763608.875180:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067684992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763608.875190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.875193:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.875197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067684992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763608.875203:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067685120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763608.875223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763608.875236:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef4000 x1876702067685120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763608.875249:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067685120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763608.875257:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.875261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.875264:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067685120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763608.875284:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067685248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763608.875304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763608.875316:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763608/real 1789763608] req@ffff936010ef4380 x1876702067685248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763608.875330:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067685248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763608.875347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763608.875351:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763608.875355:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067685248:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763609.642012:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763610.666047:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763611.690036:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763612.714067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763613.738023:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763613.993944:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763613.993959:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763613.993965:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763613.993979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.993994:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994003:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763613.994050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067685376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763613.994095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763613.994097:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763613.994102:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994112:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994125:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763613.994140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067685504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763613.994161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763613.994168:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763613.994169:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067685376:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763613.994180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994185:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994190:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994196:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763613.994214:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067685632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763613.994234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763613.994238:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763613.994247:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994255:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994263:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763613.994283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763613.994306:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067685760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763613.994334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763613.994337:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763613.994345:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061dc700 x1876702067685376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763613.994350:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994363:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994375:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994385:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763613.994394:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067685376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763613.994409:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067685888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763613.994439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763613.994445:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763613.994452:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763613.994460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763613.994465:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994474:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763613.994481:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067685376:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763613.994485:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994498:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763613.994513:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067686016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763613.994528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763613.994532:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685504:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763613.994535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763613.994542:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763613.994553:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1081 00000100:00080000:2.0:1789763613.994560:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763613.994564:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763613.994572:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763613.994619:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763613.994632:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067686144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763613.994648:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061dc380 x1876702067685504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.994679:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067685504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.994692:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.994698:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.994703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763613.994713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763613.994739:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763613.994752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061ddf80 x1876702067685632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.994776:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067685632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.994786:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.994790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.994793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763613.994834:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763613.994861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763613.994876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061de300 x1876702067685760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.994891:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067685760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.994901:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.994905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.994909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763613.994916:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763613.994937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763613.994949:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061ddc00 x1876702067685888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.994962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067685888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.994981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.994985:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.994989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067685888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763613.995010:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763613.995031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763613.995043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061dd880 x1876702067686016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.995057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067686016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.995094:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.995102:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.995110:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763613.995123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763613.995164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763613.995186:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763613/real 1789763613] req@ffff9360061df100 x1876702067686144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763613.995201:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067686144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763613.995212:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763613.995216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763613.995220:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686144:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763614.762048:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763615.786095:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763616.810027:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763617.833934:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763618.858096:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763619.113934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.113950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763619.113971:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763619.113990:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114005:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763619.114043:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763619.114092:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067686272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763619.114147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114155:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763619.114159:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763619.114169:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114178:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763619.114208:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067686272:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763619.114210:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763619.114237:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763619.114269:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067686400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763619.114289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763619.114298:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763619.114305:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114314:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763619.114322:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763619.114334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789763619.114337:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763619.114361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067686528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763619.114397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763619.114405:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763619.114414:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00100000:1.0:1789763619.114415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061df480 x1876702067686272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763619.114423:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763619.114437:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763619.114452:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763619.114466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067686272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763619.114473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067686656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763619.114513:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763619.114515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763619.114530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763619.114532:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763619.114569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067686272:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763619.114575:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114589:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763619.114605:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763619.114612:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763619.114629:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067686784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763619.114635:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686400:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763619.114664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114670:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763619.114673:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763619.114681:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114688:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763619.114696:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763619.114700:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763619.114721:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763619.114729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061dd500 x1876702067686400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763619.114743:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067686912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763619.114762:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067686400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763619.114765:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763619.114783:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763619.114791:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763619.114796:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763619.114839:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1086 00000100:00080000:2.0:1789763619.114849:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763619.114855:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763619.114864:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763619.114871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686400:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789763619.114877:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763619.114889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763619.114905:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067687040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763619.114940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763619.114985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061dca80 x1876702067686528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763619.115016:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067686528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763619.115037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763619.115046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763619.115054:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763619.115077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763619.115114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763619.115131:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061df800 x1876702067686656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763619.115146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067686656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763619.115157:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763619.115161:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763619.115165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763619.115187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763619.115209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763619.115230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061dfb80 x1876702067686784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763619.115269:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067686784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763619.115296:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763619.115303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763619.115310:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067686784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763619.115322:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763619.115354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763619.115370:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061de680 x1876702067686912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763619.115385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067686912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763619.115396:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763619.115400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763619.115404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067686912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763619.115411:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763619.115431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763619.115442:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763619/real 1789763619] req@ffff9360061dc000 x1876702067687040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763619.115456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067687040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763619.115473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763619.115477:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763619.115481:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687040:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763619.882018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763620.906072:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763621.930076:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763622.954039:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763623.978039:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763624.233958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.233974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763624.233980:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.233994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234008:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234017:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763624.234065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067687168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.234126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763624.234128:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234133:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234138:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234142:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763624.234153:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763624.234170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067687296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.234190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763624.234193:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234199:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234204:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234209:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763624.234216:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763624.234228:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067687424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.234245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763624.234247:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234254:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234260:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234268:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763624.234271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067687168:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763624.234275:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763624.234297:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067687552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.234315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763624.234317:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234323:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234326:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234331:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763624.234340:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763624.234354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067687680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234365:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763624.234370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763624.234372:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234379:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234385:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234390:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763624.234395:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763624.234407:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067687808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763624.234419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763624.234433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763624.234437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763624.234441:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763624.234449:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1092 00000100:00080000:3.0:1789763624.234455:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763624.234460:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763624.234466:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763624.234484:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067687936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763624.234558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa7b80 x1876702067687168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.234602:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067687168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.234625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.234633:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.234638:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067687168:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763624.234652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763624.234693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.234712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa5180 x1876702067687296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.234729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067687296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.234740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.234746:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.234750:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763624.234788:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763624.234883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.234909:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa4380 x1876702067687424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.234936:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067687424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.234954:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.234962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.234969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763624.234983:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763624.235016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.235032:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa7800 x1876702067687552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.235047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067687552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.235057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.235062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.235066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763624.235072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763624.235095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.235108:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa6300 x1876702067687680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.235130:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067687680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.235146:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.235152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.235159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067687680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763624.235172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763624.235215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.235234:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa6680 x1876702067687808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.235259:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067687808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.235275:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.235281:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.235288:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763624.235301:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763624.235343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763624.235361:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763624/real 1789763624] req@ffff936010aa7100 x1876702067687936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763624.235387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067687936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763624.235397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763624.235401:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763624.235405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067687936:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763625.003735:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763626.026061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763627.050073:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763628.074079:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763629.098047:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763629.353908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.353923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763629.353928:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763629.353942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.353956:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.353966:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763629.354015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067688064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763629.354056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.354061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763629.354064:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763629.354080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.354090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763629.354103:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763629.354119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067688192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763629.354134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.354138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763629.354141:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763629.354146:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354150:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.354159:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763629.354162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067688064:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763629.354166:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763629.354187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067688320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763629.354210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.354218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763629.354222:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763629.354231:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354239:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.354248:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763629.354259:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763629.354262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763629.354294:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067688448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763629.354313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.354318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763629.354322:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763629.354330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa4000 x1876702067688064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763629.354333:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354343:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.354356:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763629.354368:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763629.354378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067688064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763629.354389:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067688576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763629.354405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763629.354411:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763629.354418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763629.354421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763629.354426:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763629.354428:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067688064:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763629.354438:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354449:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763629.354451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688192:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763629.354464:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763629.354496:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763629.354525:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067688704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763629.354533:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763629.354556:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763629.354563:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763629.354569:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763629.354571:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa5500 x1876702067688192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763629.354579:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1097 00000100:00080000:3.0:1789763629.354591:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763629.354602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763629.354615:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763629.354630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067688192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763629.354638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067688832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763629.354698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.354706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.354713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763629.354749:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763629.354844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763629.354864:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa4a80 x1876702067688320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763629.354880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067688320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763629.354891:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.354895:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.354899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763629.354908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763629.354934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763629.354963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa6a00 x1876702067688448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763629.354979:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067688448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763629.354990:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.354993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.354998:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763629.355006:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763629.355029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763629.355042:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa5c00 x1876702067688576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763629.355056:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067688576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763629.355066:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.355069:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.355073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067688576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763629.355080:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067688704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763629.355100:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763629.355113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa7480 x1876702067688704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763629.355127:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067688704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763629.355136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.355140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.355143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067688704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763629.355149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067688832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763629.355168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763629.355181:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763629/real 1789763629] req@ffff936010aa4e00 x1876702067688832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763629.355193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067688832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763629.355203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763629.355206:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763629.355211:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067688832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763630.122034:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763631.146038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763632.170040:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763633.194061:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763634.218024:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763634.473969:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.473989:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763634.473997:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763634.474014:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474031:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763634.474043:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763634.474094:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067688960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763634.474132:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474138:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763634.474141:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763634.474153:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474159:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763634.474164:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474176:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763634.474193:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067689088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763634.474206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474211:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763634.474214:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763634.474243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067688960:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763634.474290:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474299:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763634.474308:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474319:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763634.474347:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763634.474352:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067689216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763634.474377:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474382:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763634.474385:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763634.474391:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474397:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763634.474403:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474413:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763634.474416:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061dd180 x1876702067688960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763634.474461:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067689344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763634.474477:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474481:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763634.474484:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763634.474490:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474496:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763634.474501:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067688960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763634.474507:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474520:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763634.474541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763634.474548:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067689472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763634.474555:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.474570:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067688960:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763634.474576:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474584:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763634.474586:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763634.474589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689088:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763634.474597:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474602:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763634.474607:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474613:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763634.474631:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067689600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763634.474646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763634.474647:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763634.474656:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763634.474660:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763634.474668:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1102 00000100:00080000:2.0:1789763634.474674:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763634.474675:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061dce00 x1876702067689088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763634.474684:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763634.474699:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763634.474712:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067689088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763634.474722:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067689728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763634.474741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.474755:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.474764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763634.474847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763634.474891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763634.474917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061dc000 x1876702067689216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763634.474943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067689216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763634.474954:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.474959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.474963:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763634.474971:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763634.475000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763634.475015:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061de680 x1876702067689344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763634.475029:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067689344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763634.475040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.475046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.475053:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763634.475064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763634.475102:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763634.475124:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061dfb80 x1876702067689472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763634.475148:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067689472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763634.475159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.475163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.475167:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763634.475174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067689600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763634.475208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763634.475230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061df800 x1876702067689600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763634.475254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067689600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763634.475271:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.475277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.475284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067689600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763634.475296:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067689728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763634.475326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763634.475341:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763634/real 1789763634] req@ffff9360061dca80 x1876702067689728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763634.475355:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067689728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763634.475366:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763634.475369:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763634.475373:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067689728:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763635.242046:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763636.266106:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763637.290045:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763638.314077:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763639.338117:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763639.593943:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.593957:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763639.593961:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763639.593975:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.593988:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763639.593997:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763639.594041:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067689856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763639.594076:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.594081:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763639.594083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763639.594088:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.594092:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763639.594097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763639.594108:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763639.594123:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067689984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763639.594153:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763639.594157:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067689856:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763639.594160:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763639.594167:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763639.594177:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.594181:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763639.594186:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763639.594194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763639.594213:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067690112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763639.594226:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.594232:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763639.594237:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763639.594245:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.594254:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763639.594258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763639.594270:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763639.594284:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763639.594314:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067690240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763639.594321:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa5880 x1876702067689856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.594367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067689856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763639.594376:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.594414:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763639.594421:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763639.594424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763639.594431:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.594442:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763639.594444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763639.594452:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763639.594465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067689856:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763639.594466:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763639.594494:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067690368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763639.594508:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.594513:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763639.594516:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763639.594524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689984:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763639.594525:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00080000:3.0:1789763639.594537:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763639.594544:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763639.594551:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763639.594572:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067690496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763639.594587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763639.594594:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763639.594601:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763639.594604:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763639.594612:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1107 00000100:00100000:1.0:1789763639.594618:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa5f80 x1876702067689984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763639.594622:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763639.594637:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763639.594650:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763639.594677:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067690624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763639.594682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067689984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.594716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.594722:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.594729:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067689984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763639.594739:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763639.594766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763639.594781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa4700 x1876702067690112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.594838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067690112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.594850:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.594855:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.594859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763639.594867:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763639.594894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763639.594908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa4e00 x1876702067690240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.594923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067690240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.594941:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.594945:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.594949:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763639.594955:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763639.594975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763639.594987:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa7480 x1876702067690368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.595000:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067690368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.595010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.595013:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.595017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763639.595042:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067690496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763639.595063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763639.595076:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa5c00 x1876702067690496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.595089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067690496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.595098:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.595101:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.595106:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067690496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763639.595113:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067690624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763639.595132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763639.595143:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763639/real 1789763639] req@ffff936010aa6a00 x1876702067690624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763639.595157:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067690624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763639.595166:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763639.595170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763639.595173:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067690624:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763640.362005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763641.386087:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763642.409996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763643.434065:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763644.458035:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763644.713906:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763644.713920:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763644.713926:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763644.713940:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.713954:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.713963:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763644.714016:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067690752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763644.714070:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763644.714078:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763644.714084:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763644.714093:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714101:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714109:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714123:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763644.714146:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067690880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763644.714164:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763644.714169:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763644.714174:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763644.714177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067690752:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763644.714184:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714198:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714206:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763644.714245:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067691008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763644.714267:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763644.714282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763644.714294:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763644.714298:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763644.714306:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714313:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714320:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714329:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763644.714349:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067691136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763644.714385:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763644.714393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa4a80 x1876702067690752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763644.714394:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763644.714401:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763644.714417:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714427:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714437:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714445:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763644.714453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067690752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763644.714469:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067691264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763644.714476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763644.714493:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763644.714494:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763644.714504:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763644.714511:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763644.714512:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067690752:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763644.714522:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714530:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714535:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714541:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763644.714553:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067691392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763644.714565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763644.714576:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763644.714582:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763644.714585:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763644.714593:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1112 00000100:00080000:3.0:1789763644.714601:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763644.714605:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763644.714613:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763644.714616:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763644.714637:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067691520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763644.714648:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa5500 x1876702067690880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.714702:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067690880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.714721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.714728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.714735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067690880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763644.714749:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763644.714848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763644.714870:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa4000 x1876702067691008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.714886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067691008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.714897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.714901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.714905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763644.714912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763644.714940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763644.714953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa7100 x1876702067691136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.714976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067691136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.714986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.714989:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.714993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763644.714999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763644.715020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763644.715032:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa6680 x1876702067691264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.715045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067691264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.715055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.715058:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.715062:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763644.715088:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067691392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763644.715109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763644.715122:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa6300 x1876702067691392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.715135:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067691392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.715152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.715156:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.715159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067691392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763644.715166:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067691520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763644.715185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763644.715197:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763644/real 1789763644] req@ffff936010aa7800 x1876702067691520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763644.715210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067691520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763644.715219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763644.715223:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763644.715227:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067691520:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763645.482032:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763646.506051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763647.530022:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763648.554010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763649.577915:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763649.833911:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.833924:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763649.833930:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.833943:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.833971:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.833977:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763649.834010:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067691648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763649.834032:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834035:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763649.834037:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834040:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834042:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834044:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834051:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763649.834059:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067691776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763649.834066:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763649.834069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834071:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834073:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834075:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834078:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763649.834084:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067691904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763649.834091:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834093:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763649.834094:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834098:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834100:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834102:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834106:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763649.834111:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067691648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763649.834116:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067692032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763649.834123:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834125:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763649.834126:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834129:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834131:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834133:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834136:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763649.834145:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067692160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763649.834151:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834153:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763649.834154:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834156:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834158:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834161:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834164:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763649.834172:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067692288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763649.834175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763649.834183:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763649.834186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763649.834188:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763649.834192:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1117 00000100:00080000:3.0:1789763649.834195:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763649.834197:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763649.834201:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763649.834209:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067692416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763649.834230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa4380 x1876702067691648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067691648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834308:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834314:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834351:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067691648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763649.834362:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763649.834398:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834407:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa5180 x1876702067691776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834415:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067691776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834421:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834425:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763649.834429:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763649.834448:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa7b80 x1876702067691904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834462:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067691904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834467:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834469:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834471:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067691904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763649.834494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763649.834506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa7800 x1876702067692032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067692032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763649.834563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763649.834584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834593:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa6300 x1876702067692160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067692160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834626:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834628:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834630:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763649.834634:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067692288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763649.834647:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834653:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa6680 x1876702067692288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067692288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834668:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067692288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763649.834673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067692416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763649.834693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763649.834699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763649/real 1789763649] req@ffff936010aa7100 x1876702067692416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763649.834705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067692416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763649.834710:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763649.834712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763649.834714:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067692416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763650.602058:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763651.626071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763652.650017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763653.674023:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763654.698052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763654.954018:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954029:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763654.954036:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763654.954049:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954067:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954077:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763654.954135:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067692544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763654.954172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954177:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763654.954180:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763654.954190:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954209:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954261:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763654.954307:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067692672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763654.954337:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954345:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763654.954352:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067692544:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763654.954354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763654.954380:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954389:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954397:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954406:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763654.954428:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067692800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763654.954443:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954448:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763654.954450:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763654.954458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763654.954465:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954477:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954484:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954493:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763654.954514:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067692928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763654.954530:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763654.954534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061dd500 x1876702067692544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763654.954540:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763654.954547:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763654.954572:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954580:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954600:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954608:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763654.954617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067692544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763654.954630:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067693056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763654.954659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763654.954660:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954681:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763654.954686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763654.954689:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763654.954698:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067692544:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763654.954699:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954713:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954723:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954730:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763654.954742:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067693184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763654.954752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692672:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763654.954759:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763654.954765:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763654.954767:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763654.954774:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1122 00000100:00080000:2.0:1789763654.954780:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763654.954785:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763654.954792:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763654.954867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763654.954874:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067693312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763654.954900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061df480 x1876702067692672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.954932:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067692672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.954945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.954952:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.954960:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763654.954973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763654.955000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763654.955012:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061df100 x1876702067692800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.955026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067692800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.955037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.955042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.955046:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763654.955057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763654.955079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763654.955090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061dd880 x1876702067692928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.955101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067692928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.955110:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.955113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.955117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067692928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763654.955122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763654.955140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763654.955150:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061ddc00 x1876702067693056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.955161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067693056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.955169:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.955172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.955176:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763654.955194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067693184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763654.955213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763654.955223:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061de300 x1876702067693184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.955254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067693184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.955263:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.955266:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.955269:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067693184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763654.955275:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067693312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763654.955321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763654.955333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763654/real 1789763654] req@ffff9360061ddf80 x1876702067693312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763654.955356:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067693312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763654.955364:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763654.955368:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763654.955371:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067693312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763655.722127:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763656.746117:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763657.770083:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763658.794015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763659.818032:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763660.073992:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763660.074022:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074037:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074050:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074060:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763660.074117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067693440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763660.074174:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074179:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074183:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074187:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763660.074200:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763660.074214:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067693568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763660.074236:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074242:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074273:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074280:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763660.074287:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763660.074311:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067693696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763660.074330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067693440:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763660.074335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763660.074341:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074354:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074358:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074362:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763660.074369:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763660.074384:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067693824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763660.074403:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074407:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074411:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074415:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763660.074423:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763660.074441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763660.074442:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067693952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763660.074468:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074474:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074478:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074482:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763660.074489:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763660.074515:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa4000 x1876702067693440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763660.074515:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067694080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763660.074557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763660.074569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763660.074572:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763660.074579:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1127 00000100:00080000:3.0:1789763660.074584:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763660.074592:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763660.074597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067693440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763660.074599:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763660.074616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067694208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763660.074638:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.074656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.074674:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067693440:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763660.074689:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763660.074727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.074745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa5500 x1876702067693568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.074771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067693568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.074782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.074786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.074790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763660.074893:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763660.074944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.074970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa4a80 x1876702067693696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.074995:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067693696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.075015:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.075021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.075027:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763660.075039:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763660.075070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.075084:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa6a00 x1876702067693824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.075099:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067693824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.075109:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.075113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.075117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763660.075123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763660.075148:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.075160:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa5c00 x1876702067693952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.075179:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067693952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.075194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.075201:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.075218:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067693952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763660.075230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763660.075305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.075325:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa7480 x1876702067694080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.075356:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067694080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.075373:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.075379:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.075385:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763660.075398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763660.075439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763660.075461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763660/real 1789763660] req@ffff936010aa4e00 x1876702067694208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763660.075477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067694208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763660.075487:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763660.075491:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763660.075495:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763660.842122:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763661.866061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763662.890069:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763663.914134:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763664.938088:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763665.193976:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.193990:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763665.193995:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194007:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194019:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194028:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763665.194082:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067694336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194133:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.194138:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763665.194140:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194145:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194150:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194154:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763665.194183:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763665.194194:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067694464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194204:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.194226:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763665.194228:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194233:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194237:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194240:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763665.194246:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763665.194259:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067694592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194286:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.194290:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763665.194292:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194297:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194301:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194306:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763665.194315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067694336:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763665.194316:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763665.194340:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067694720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194353:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.194358:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763665.194360:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194365:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194369:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194373:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763665.194379:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763665.194391:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067694848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194401:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763665.194405:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763665.194407:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194411:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194416:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194420:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763665.194426:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763665.194439:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067694976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763665.194451:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763665.194459:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694464:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763665.194460:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763665.194485:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763665.194493:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1133 00000100:00080000:3.0:1789763665.194498:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763665.194502:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763665.194508:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763665.194536:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067695104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763665.194540:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763665.194636:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763665.194668:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763665.194708:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763665.194728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067695104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763665.962063:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763666.986010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763668.010063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763668.266130:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763668.266207:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763668.266232:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266278:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266287:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266294:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266304:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266312:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763668.266319:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763668.266358:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa4e00 x1876702067694720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067694720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266431:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763668.266443:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa7480 x1876702067694848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266459:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067694848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266474:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763668.266487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa5c00 x1876702067694976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067694976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067694976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763668.266540:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa6a00 x1876702067695104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266553:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067695104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266578:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266582:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067695104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763668.266591:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa4700 x1876702067694336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266608:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067694336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266619:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266628:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067694336:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763668.266637:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa5f80 x1876702067694464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067694464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266673:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763668.266681:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763665/real 1789763668] req@ffff936010aa5880 x1876702067694592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763668.266694:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067694592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763668.266704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763668.266709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763668.266713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067694592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763669.033956:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763670.058058:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763670.313903:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.313919:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763670.313924:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.313938:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.313953:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.313963:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763670.314010:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067695232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763670.314047:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314052:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763670.314054:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314059:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314063:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.314067:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314079:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763670.314093:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067695360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763670.314107:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314112:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763670.314116:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314121:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314125:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.314130:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314137:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763670.314153:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067695232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763670.314154:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067695488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763670.314209:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314214:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763670.314216:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314222:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314227:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.314232:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314239:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763670.314257:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067695616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763670.314269:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314272:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763670.314275:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314279:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314284:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.314288:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314292:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763670.314304:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067695744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763670.314314:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314322:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:1.0:1789763670.314327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763670.314328:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314343:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314349:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763670.314356:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314363:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763670.314383:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067695872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763670.314409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa4a80 x1876702067695232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763670.314420:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763670.314436:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763670.314443:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763670.314456:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1138 00000100:00080000:3.0:1789763670.314465:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763670.314474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067695232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763670.314482:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763670.314501:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763670.314512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.314528:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763670.314536:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067696000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763670.314541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067695232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763670.314570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763670.314642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.314672:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa5500 x1876702067695360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.314699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067695360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.314715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.314734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.314741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763670.314787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763670.314890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.314917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa4000 x1876702067695488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.314943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067695488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.314960:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.314967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.314974:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763670.314986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763670.315030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.315054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa7100 x1876702067695616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.315080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067695616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.315096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.315102:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.315109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763670.315121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763670.315162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.315219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa6680 x1876702067695744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.315247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067695744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.315265:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.315271:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.315277:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067695744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763670.315291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067695872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763670.315338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.315363:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa6300 x1876702067695872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.315402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067695872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.315417:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.315423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.315430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067695872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763670.315441:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763670.315482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763670.315507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763670/real 1789763670] req@ffff936010aa7800 x1876702067696000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763670.315531:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067696000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763670.315547:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763670.315554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763670.315561:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763671.081937:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763672.106021:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763673.130022:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763674.154068:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763675.178023:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763675.433902:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763675.433917:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763675.433923:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.433936:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.433949:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.433957:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763675.434003:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067696128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763675.434044:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763675.434052:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763675.434056:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.434063:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434072:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434081:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763675.434099:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763675.434121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067696128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763675.434151:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067696256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763675.434174:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763675.434179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763675.434182:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.434188:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434195:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434200:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763675.434208:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763675.434227:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067696384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763675.434240:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763675.434249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763675.434250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763675.434257:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.434265:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434271:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434277:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763675.434284:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763675.434300:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067696512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763675.434324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa7b80 x1876702067696128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763675.434325:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763675.434341:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763675.434347:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.434362:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434373:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434383:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763675.434390:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763675.434397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067696128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763675.434410:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067696640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763675.434420:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763675.434429:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763675.434435:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763675.434438:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763675.434441:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763675.434446:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434450:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434464:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763675.434469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067696128:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763675.434473:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763675.434495:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067696768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763675.434506:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763675.434510:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763675.434513:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763675.434520:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763675.434526:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1143 00000100:00080000:3.0:1789763675.434536:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763675.434542:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763675.434548:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763675.434563:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067696896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763675.434575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.434605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa5180 x1876702067696256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.434632:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067696256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.434650:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.434657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.434664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763675.434678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763675.434721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.434745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa4380 x1876702067696384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.434771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067696384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.434848:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.434854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.434858:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763675.434880:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763675.434914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.434931:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa7800 x1876702067696512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.434946:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067696512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.434956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.434960:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.434964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763675.434971:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763675.434991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.435003:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa6300 x1876702067696640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.435017:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067696640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.435026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.435029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.435034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067696640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763675.435041:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763675.435062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.435074:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa6680 x1876702067696768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.435087:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067696768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.435097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.435100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.435104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763675.435130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763675.435191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763675.435213:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763675/real 1789763675] req@ffff936010aa7100 x1876702067696896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763675.435237:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067696896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763675.435254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763675.435262:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763675.435268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067696896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763676.202154:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763677.226087:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763678.249918:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763679.274068:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763680.298074:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763680.553924:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.553939:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763680.553944:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.553957:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.553970:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.553979:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763680.554025:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067697024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763680.554061:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554067:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763680.554069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554074:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554078:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.554083:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554112:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763680.554128:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067697152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763680.554152:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554156:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763680.554159:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554167:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554172:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.554177:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554182:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763680.554197:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067697280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763680.554212:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067697024:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763680.554217:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554228:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763680.554233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554239:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554242:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.554247:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554253:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763680.554270:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067697408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763680.554282:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554287:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763680.554289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554294:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554299:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.554305:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554311:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763680.554323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763680.554329:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067697536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763680.554345:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554350:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763680.554353:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554357:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554361:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763680.554366:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763680.554387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa4000 x1876702067697024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763680.554395:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067697664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763680.554452:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763680.554457:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763680.554461:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763680.554466:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1148 00000100:00080000:3.0:1789763680.554472:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763680.554476:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067697024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763680.554480:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763680.554505:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763680.554510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.554523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763680.554525:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067697792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763680.554536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067697024:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763680.554563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763680.554607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.554628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa5500 x1876702067697152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.554644:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067697152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.554654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.554659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.554663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763680.554694:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763680.554729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.554753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa4a80 x1876702067697280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.554780:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067697280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.554838:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.554845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.554850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763680.554859:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763680.554892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.554908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa5880 x1876702067697408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.554923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067697408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.554933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.554937:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.554941:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763680.554947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763680.554967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.554979:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa5f80 x1876702067697536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.554993:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067697536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.555011:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.555015:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.555019:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067697536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763680.555025:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067697664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763680.555044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.555056:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa4700 x1876702067697664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.555069:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067697664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.555079:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.555082:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.555086:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067697664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763680.555092:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067697792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763680.555156:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763680.555178:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763680/real 1789763680] req@ffff936010aa6a00 x1876702067697792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763680.555202:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067697792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763680.555219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763680.555225:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763680.555233:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067697792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763681.322021:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763682.346024:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763683.370020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763684.394025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763685.417999:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763685.673901:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.673916:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763685.673922:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.673936:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:3.0:1789763685.673948:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.673957:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763685.674004:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067697920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763685.674042:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674049:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763685.674053:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674080:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:3.0:1789763685.674087:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.674095:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763685.674112:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763685.674136:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067697920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763685.674136:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067698048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763685.674157:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674164:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763685.674167:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674174:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:3.0:1789763685.674180:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.674187:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763685.674195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763685.674213:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067698176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763685.674228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763685.674230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674237:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763685.674240:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674246:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:3.0:1789763685.674251:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.674258:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763685.674285:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763685.674288:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa5c00 x1876702067697920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763685.674305:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067698304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763685.674318:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067697920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763685.674322:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674329:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763685.674332:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674338:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:1.0:1789763685.674340:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763685.674344:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763685.674347:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763685.674351:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674355:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067697920:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763685.674358:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763685.674373:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067698432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763685.674391:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674398:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763685.674401:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674408:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00100000:1.0:1789763685.674409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698048:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763685.674414:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.674421:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763685.674429:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763685.674446:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067698560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763685.674450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763685.674466:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763685.674473:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763685.674474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa7480 x1876702067698048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763685.674477:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763685.674484:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1153 00000100:00080000:3.0:1789763685.674489:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763685.674496:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674500:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067698048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763685.674504:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763685.674517:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763685.674521:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067698688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763685.674539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.674546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763685.674559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763685.674599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa4e00 x1876702067698176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763685.674656:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067698176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763685.674667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763685.674671:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.674675:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763685.674683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763685.674711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa6a00 x1876702067698304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763685.674739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067698304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763685.674749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763685.674752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.674756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763685.674762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763685.674782:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674795:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa4700 x1876702067698432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763685.674847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067698432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763685.674857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763685.674861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.674865:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763685.674894:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067698560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763685.674919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763685.674933:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa5f80 x1876702067698560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763685.674948:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067698560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763685.674957:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763685.674961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.674965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067698560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763685.674972:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067698688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763685.674991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763685.675003:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763685/real 1789763685] req@ffff936010aa5880 x1876702067698688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763685.675016:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067698688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763685.675034:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763685.675038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763685.675042:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067698688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763686.442047:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763687.466076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763688.490112:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763689.514050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763690.538082:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763690.793972:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.793983:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763690.793989:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763690.794005:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.794044:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794059:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763690.794115:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067698816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763690.794164:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.794170:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763690.794173:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763690.794181:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.794187:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794195:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763690.794209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763690.794239:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067698944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763690.794242:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067698816:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763690.794274:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.794280:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763690.794283:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763690.794291:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.794298:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794304:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763690.794312:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763690.794327:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067699072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763690.794341:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.794345:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763690.794350:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763690.794358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763690.794362:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.794382:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794388:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763690.794396:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763690.794436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa4a80 x1876702067698816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763690.794473:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067698816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763690.794496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763690.794505:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763690.794506:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067699200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763690.794521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067698816:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763690.794531:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.794541:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763690.794544:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763690.794552:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.794558:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794563:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763690.794571:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763690.794581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763690.794593:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067699328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763690.794607:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763690.794628:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763690.794631:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763690.794665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.794696:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa5500 x1876702067698944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763690.794732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067698944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763690.794744:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763690.794749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763690.794757:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067698944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763690.794771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763690.794869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.794899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa4000 x1876702067699072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763690.794929:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00100000:2.0:1789763690.794931:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067699072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763690.794940:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.794957:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789763690.794970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763690.794971:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763690.794991:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763690.795005:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067699456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763690.795008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763690.795068:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763690.795126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.795160:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa7100 x1876702067699200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763690.795175:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763690.795183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067699200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763690.795186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763690.795194:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763690.795210:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1158 00000100:00080000:3.0:1789763690.795222:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763690.795235:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789763690.795237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763690.795245:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763690.795253:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763690.795265:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763690.795275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067699584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763690.795324:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763690.795371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.795393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa6680 x1876702067699328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763690.795410:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067699328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763690.795422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763690.795427:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763690.795432:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763690.795442:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067699456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763690.795467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.795486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa6300 x1876702067699456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763690.795502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067699456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763690.795511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763690.795514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763690.795519:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067699456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763690.795526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067699584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763690.795546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763690.795558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763690/real 1789763690] req@ffff936010aa7800 x1876702067699584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763690.795572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067699584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763690.795581:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763690.795584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763690.795588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067699584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763691.562007:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763692.586074:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763693.610037:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763694.634049:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763695.658049:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763695.913931:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.913946:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763695.913951:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.913965:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.913978:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914020:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763695.914065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067699712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763695.914112:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.914117:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763695.914120:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914125:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.914129:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914135:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914147:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763695.914164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067699840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763695.914179:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.914184:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763695.914187:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914196:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00100000:1.0:1789763695.914201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067699712:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763695.914203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914215:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914223:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763695.914241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067699968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763695.914255:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.914259:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763695.914262:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914267:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.914272:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914287:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914294:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763695.914302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763695.914318:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067700096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763695.914332:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.914337:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763695.914340:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914346:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.914353:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763695.914362:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061dc380 x1876702067699712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763695.914364:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763695.914397:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067700224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763695.914411:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763695.914416:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763695.914422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067699712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763695.914422:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914438:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.914446:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914458:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914471:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763695.914478:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.914492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763695.914501:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067700352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763695.914505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067699712:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763695.914532:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763695.914533:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699840:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763695.914542:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763695.914549:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763695.914555:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1163 00000100:00080000:2.0:1789763695.914560:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763695.914566:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763695.914574:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763695.914588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763695.914597:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067700480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763695.914619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061dc700 x1876702067699840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.914649:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067699840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.914660:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.914664:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.914668:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763695.914698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763695.914722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763695.914735:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061ddf80 x1876702067699968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.914749:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067699968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.914765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.914769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.914773:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067699968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763695.914779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763695.914847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763695.914863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061de300 x1876702067700096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.914878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067700096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.914897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.914900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.914905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763695.914911:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763695.914935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763695.914948:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061ddc00 x1876702067700224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.914961:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067700224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.914971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.914974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.914977:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763695.915024:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067700352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763695.915053:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763695.915068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061dd880 x1876702067700352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.915083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067700352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.915092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.915096:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.915101:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067700352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763695.915108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067700480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763695.915128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763695.915140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763695/real 1789763695] req@ffff9360061df100 x1876702067700480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763695.915153:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067700480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763695.915163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763695.915166:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763695.915169:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067700480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763696.682080:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763697.706052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763698.730050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763699.754018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763700.778146:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763701.033978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.033995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763701.034001:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034020:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034038:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034050:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763701.034108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067700608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034153:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.034158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763701.034161:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034167:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034172:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034178:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763701.034212:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067700736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.034233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763701.034236:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034248:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00100000:1.0:1789763701.034251:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067700608:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763701.034254:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034275:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034283:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763701.034303:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067700864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.034321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763701.034324:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034329:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034332:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034338:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034344:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763701.034359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067700992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034373:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.034378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763701.034380:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034383:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034387:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034391:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034398:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763701.034408:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763701.034416:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067701120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763701.034435:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763701.034438:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034445:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034452:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034458:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034464:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763701.034491:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067701248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763701.034505:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763701.034509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020dafb80 x1876702067700608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763701.034511:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763701.034517:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763701.034534:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1168 00000100:00080000:0.0:1789763701.034544:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763701.034561:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763701.034581:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763701.034609:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067701376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763701.034626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067700608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.034649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.034659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.034667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067700608:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763701.034709:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763701.034749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.034768:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020dac000 x1876702067700736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.034784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067700736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.034795:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.034848:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.034853:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763701.034862:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763701.034890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.034905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020daed80 x1876702067700864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.034920:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067700864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.034931:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.034936:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.034940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763701.034978:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763701.035011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.035026:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020dae680 x1876702067700992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.035042:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067700992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.035053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.035057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.035061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067700992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763701.035067:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763701.035089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.035101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020dac380 x1876702067701120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.035115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067701120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.035125:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.035128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.035132:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763701.035139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067701248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763701.035160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.035172:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020daf480 x1876702067701248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.035186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067701248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.035204:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.035208:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.035212:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067701248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763701.035219:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067701376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763701.035240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763701.035252:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763701/real 1789763701] req@ffff936020dad880 x1876702067701376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763701.035265:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067701376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763701.035274:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763701.035277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763701.035281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067701376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763701.802079:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763702.826041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763703.850044:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763704.874061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763705.898067:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763706.153901:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763706.153925:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763706.153929:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.153949:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.153959:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.153966:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763706.154011:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067701504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763706.154032:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763706.154036:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763706.154037:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154040:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154043:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154046:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154055:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763706.154065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067701632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763706.154072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067701504:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763706.154076:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763706.154080:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763706.154082:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154085:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154087:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154090:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154093:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763706.154101:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067701760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763706.154108:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763706.154111:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763706.154112:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154115:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154117:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154119:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154123:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763706.154134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763706.154134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067701888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763706.154144:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763706.154147:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763706.154148:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154151:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154154:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154156:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154159:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763706.154166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067702016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763706.154172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763706.154174:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa4380 x1876702067701504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763706.154176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763706.154179:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154187:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154192:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154195:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154199:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763706.154201:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067701504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763706.154209:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067702144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763706.154213:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763706.154220:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763706.154221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763706.154227:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763706.154230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067701504:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763706.154231:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763706.154239:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1173 00000100:00080000:3.0:1789763706.154242:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763706.154245:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763706.154248:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763706.154255:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067702272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763706.154258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763706.154294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154310:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa5180 x1876702067701632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154323:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067701632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154332:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154336:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763706.154347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763706.154368:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154377:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa7b80 x1876702067701760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067701760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154391:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154393:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763706.154398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763706.154411:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154417:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa7800 x1876702067701888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154424:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067701888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154429:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067701888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763706.154436:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763706.154474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa6300 x1876702067702016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067702016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763706.154542:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067702144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763706.154571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa6680 x1876702067702144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067702144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067702144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763706.154620:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067702272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763706.154633:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763706.154639:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763706/real 1789763706] req@ffff936010aa7100 x1876702067702272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763706.154647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067702272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763706.154652:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763706.154654:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763706.154655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067702272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763706.921929:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763707.946052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763708.970066:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763709.993973:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763711.017925:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763711.273993:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274009:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763711.274014:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274028:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274041:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274050:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763711.274111:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067702400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763711.274132:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274135:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763711.274136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274139:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274141:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274144:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274152:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763711.274166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067702528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763711.274170:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067702400:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763711.274181:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274184:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763711.274185:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274189:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274193:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274196:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274199:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763711.274208:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067702656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763711.274215:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274217:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763711.274219:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274223:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274228:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763711.274239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763711.274242:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274249:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763711.274273:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067702784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763711.274283:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763711.274286:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061df480 x1876702067702400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763711.274288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763711.274292:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274300:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274306:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274312:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274317:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763711.274320:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067702400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274330:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763711.274331:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067702912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763711.274335:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274343:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067702400:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763711.274351:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274357:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763711.274358:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274362:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1179 00000100:00080000:2.0:1789763711.274364:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274367:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274369:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763711.274376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763711.274378:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067703040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763711.274390:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763711.274393:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763711.274394:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763711.274398:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1179 00000800:00000100:1.0:1789763711.274402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763711.274403:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763711.274408:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763711.274412:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763711.274417:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061dd500 x1876702067702528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763711.274428:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067703168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763711.274429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067702528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763711.274467:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763711.274481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763711.274488:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061dca80 x1876702067702656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763711.274495:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067702656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274505:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274507:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274509:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763711.274512:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763711.274523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763711.274530:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061df800 x1876702067702784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763711.274541:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067702784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274556:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763711.274562:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763711.274582:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763711.274592:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061dfb80 x1876702067702912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763711.274600:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067702912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274610:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067702912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763711.274622:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763711.274645:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763711.274652:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061de680 x1876702067703040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763711.274659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067703040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763711.274672:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763711.274681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763711.274687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763711/real 1789763711] req@ffff9360061dc000 x1876702067703168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763711.274694:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067703168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763711.274699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763711.274701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763711.274703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763712.042039:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763713.066083:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763714.089947:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763715.114048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763716.138056:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763716.393980:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.393995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763716.394001:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394015:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394028:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394038:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763716.394089:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067703296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763716.394127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.394132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763716.394135:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394140:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394160:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763716.394176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067703424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763716.394200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.394205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763716.394207:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394213:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394217:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763716.394226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067703296:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763716.394227:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394239:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763716.394254:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067703552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763716.394267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.394272:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763716.394274:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394279:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394283:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394288:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394293:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763716.394308:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067703680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763716.394319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.394323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763716.394325:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394328:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394337:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394342:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763716.394357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763716.394367:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067703808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763716.394381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763716.394384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763716.394386:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763716.394391:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394396:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394401:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394409:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763716.394419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020dac700 x1876702067703296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763716.394435:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067703936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763716.394454:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067703296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763716.394464:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763716.394479:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.394485:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.394490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067703296:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763716.394501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763716.394510:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763716.394511:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703424:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763716.394532:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1184 00000100:00080000:0.0:1789763716.394540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763716.394547:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763716.394556:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763716.394579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763716.394580:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067704064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763716.394617:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020daf100 x1876702067703424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.394647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067703424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.394685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.394693:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.394701:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763716.394750:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763716.394785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763716.394871:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020dad500 x1876702067703552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.394890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067703552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.394901:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.394905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.394910:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763716.394918:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763716.394947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763716.394962:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020dadf80 x1876702067703680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.394976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067703680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.394987:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.394990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.394994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763716.395001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763716.395020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763716.395033:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020daf800 x1876702067703808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.395046:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067703808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.395056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.395060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.395064:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067703808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763716.395070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763716.395090:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763716.395102:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020dae300 x1876702067703936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.395115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067703936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.395124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.395130:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.395136:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067703936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763716.395148:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763716.395184:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763716.395207:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763716/real 1789763716] req@ffff936020dad180 x1876702067704064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763716.395233:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067704064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763716.395250:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763716.395255:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763716.395260:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763717.162026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763718.186035:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763719.210057:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763720.234070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763721.258051:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763721.513948:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.513958:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763721.513963:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.513974:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.513988:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.513996:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763721.514042:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067704192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763721.514070:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514073:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763721.514074:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514078:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514080:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514084:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514094:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763721.514104:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067704320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763721.514111:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514114:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763721.514115:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514118:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514120:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514122:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514127:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763721.514133:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067704192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763721.514137:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067704448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763721.514146:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514148:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763721.514149:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514151:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514153:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514156:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514159:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763721.514166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067704576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763721.514172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514175:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763721.514177:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514179:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514182:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514184:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514187:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763721.514194:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067704704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763721.514199:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514201:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763721.514202:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514205:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514208:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514211:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514215:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763721.514230:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067704832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763721.514234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763721.514242:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763721.514245:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763721.514247:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763721.514251:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1189 00000100:00080000:3.0:1789763721.514254:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763721.514257:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763721.514261:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763721.514270:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067704960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763721.514291:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa4000 x1876702067704192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514308:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067704192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514319:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514322:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514325:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067704192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763721.514331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763721.514348:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514357:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa5500 x1876702067704320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514365:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067704320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514372:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763721.514398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763721.514412:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa4a80 x1876702067704448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514427:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067704448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514432:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514436:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763721.514439:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763721.514452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa5880 x1876702067704576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067704576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763721.514478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763721.514489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514495:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa5f80 x1876702067704704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514507:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067704704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514514:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067704704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763721.514527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763721.514548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514561:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa4700 x1876702067704832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514573:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067704832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763721.514588:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763721.514603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763721.514610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763721/real 1789763721] req@ffff936010aa6a00 x1876702067704960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763721.514617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067704960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763721.514627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763721.514629:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763721.514631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067704960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763722.282097:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763723.306030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763724.330055:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763725.354005:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763726.378052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763726.633946:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.633966:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763726.633973:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763726.633992:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00080000:3.0:1789763726.634008:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763726.634023:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763726.634082:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067705088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763726.634150:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.634159:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763726.634163:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763726.634175:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00080000:3.0:1789763726.634184:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763726.634193:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763726.634212:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763726.634237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067705088:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763726.634248:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067705216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763726.634280:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.634288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763726.634294:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763726.634333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763726.634392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa4e00 x1876702067705088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.634431:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00100000:0.0:1789763726.634441:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067705088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.634444:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763726.634462:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789763726.634470:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763726.634475:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763726.634481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.634490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067705088:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763726.634525:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763726.634559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763726.634576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa7480 x1876702067705216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763726.634606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067705216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763726.634618:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763726.634623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.634628:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763726.634743:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067705344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763726.634953:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.634962:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789763726.634968:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705344:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763726.634971:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763726.634992:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00080000:3.0:1789763726.635005:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763726.635017:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763726.635026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763726.635032:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763726.635053:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa5c00 x1876702067705344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763726.635073:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067705472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763726.635094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067705344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.635106:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.635127:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763726.635134:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763726.635136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763726.635154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763726.635157:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00100000:0.0:1789763726.635170:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705344:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763726.635173:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763726.635189:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763726.635199:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763726.635207:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763726.635233:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067705600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763726.635252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763726.635253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.635264:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763726.635270:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763726.635278:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa7480 x1876702067705472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.635287:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00080000:3.0:1789763726.635307:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763726.635315:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067705472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.635328:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789763726.635336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763726.635346:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763726.635348:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.635359:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763726.635381:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763726.635392:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067705728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763726.635416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763726.635421:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763726.635431:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763726.635438:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa4e00 x1876702067705600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.635441:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763726.635461:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1194 00000100:00080000:3.0:1789763726.635476:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763726.635490:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067705600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763726.635492:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763726.635511:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763726.635513:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763726.635526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.635533:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067705600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763726.635551:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067705856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763726.635556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067705728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763726.635598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763726.635619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa6a00 x1876702067705728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763726.635637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067705728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763726.635647:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763726.635652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.635659:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067705728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763726.635673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067705856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763726.635694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763726.635706:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763726/real 1789763726] req@ffff936010aa4700 x1876702067705856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763726.635720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067705856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763726.635737:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763726.635741:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763726.635745:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067705856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763727.402080:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763728.426051:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763729.450064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763730.474011:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763731.498040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763731.753921:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.753936:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763731.753941:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.753955:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.753969:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.753978:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763731.754028:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067705984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754069:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754074:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763731.754077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754082:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754086:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.754090:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754102:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763731.754116:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067706112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763731.754136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754141:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754144:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.754148:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754153:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763731.754164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067706240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754185:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754190:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763731.754193:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754198:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754201:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.754205:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754212:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763731.754218:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067705984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763731.754229:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067706368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754244:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754248:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763731.754251:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754255:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754259:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.754264:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754270:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763731.754286:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067706496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754298:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754303:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763731.754305:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754310:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754315:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763731.754320:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754326:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763731.754338:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067706624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763731.754349:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763731.754363:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763731.754366:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763731.754371:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1199 00000100:00080000:3.0:1789763731.754376:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763731.754385:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706112:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763731.754388:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763731.754406:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763731.754439:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067706752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763731.754467:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763731.754523:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763731.754553:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763731.754578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067706624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763731.754601:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067706752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763732.522054:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763733.546001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763734.569955:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763734.826185:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789763734.826245:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763734.826266:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826314:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826324:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826335:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826353:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826371:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763734.826393:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789763734.826399:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa5500 x1876702067706368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826450:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067706368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763734.826503:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa4000 x1876702067706496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067706496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826546:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826559:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763734.826573:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa7100 x1876702067706624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067706624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826611:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826617:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067706624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763734.826630:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa6680 x1876702067706752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826654:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067706752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067706752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763734.826736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa5f80 x1876702067705984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067705984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826796:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067705984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763734.826875:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa5880 x1876702067706112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826894:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067706112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826905:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826910:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826914:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763734.826922:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763731/real 1789763734] req@ffff936010aa4a80 x1876702067706240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763734.826938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067706240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763734.826955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763734.826962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763734.826969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067706240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763735.594030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763736.618036:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763736.873995:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763736.874015:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874029:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763736.874053:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763736.874097:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067706880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763736.874140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763736.874148:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874153:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874158:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763736.874163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763736.874176:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763736.874194:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067707008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763736.874204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067706880:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763736.874215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763736.874225:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874235:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763736.874240:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763736.874247:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763736.874261:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067707136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763736.874275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763736.874282:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874286:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874303:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763736.874308:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763736.874315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763736.874321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763736.874344:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067707264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763736.874360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763736.874371:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874380:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874388:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763736.874398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020daea00 x1876702067706880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763736.874404:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763736.874421:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763736.874459:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067707392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763736.874464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067706880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763736.874521:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763736.874532:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874544:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:1.0:1789763736.874556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763736.874559:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763736.874575:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763736.874579:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763736.874593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067706880:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763736.874598:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763736.874648:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067707520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763736.874668:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707008:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763736.874707:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763736.874717:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763736.874721:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763736.874730:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1204 00000100:00080000:0.0:1789763736.874737:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763736.874744:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763736.874754:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763736.874775:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067707648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763736.875014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020dace00 x1876702067707008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875070:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067707008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875088:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875095:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763736.875115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763736.875159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875183:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020dadc00 x1876702067707136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067707136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875232:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763736.875256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763736.875298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875320:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020dad180 x1876702067707264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067707264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875368:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875379:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763736.875391:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763736.875430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020dae300 x1876702067707392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875476:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067707392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875493:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875508:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763736.875545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067707520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763736.875588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020daf800 x1876702067707520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067707520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067707520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763736.875707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067707648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763736.875750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763736.875774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763736/real 1789763736] req@ffff936020dadf80 x1876702067707648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763736.875835:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067707648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763736.875855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763736.875862:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763736.875869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067707648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763737.642077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763738.665921:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763739.689990:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763740.714029:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763741.738020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763741.993889:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.993904:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763741.993909:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.993923:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.993935:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.993943:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763741.994006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067707776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763741.994042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.994050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763741.994054:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.994062:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.994069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.994077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763741.994093:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763741.994119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067707904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763741.994143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.994148:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763741.994151:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.994158:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.994164:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.994174:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763741.994177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067707776:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763741.994183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763741.994203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067708032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763741.994219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.994224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763741.994226:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.994232:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.994259:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.994267:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763741.994274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789763741.994284:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763741.994315:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067708160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763741.994334:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020dad500 x1876702067707776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763741.994340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.994356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763741.994365:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.994374:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00100000:1.0:1789763741.994382:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067707776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763741.994384:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.994398:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763741.994415:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763741.994416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.994430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.994438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067707776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763741.994444:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067708288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763741.994456:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707904:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763741.994470:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763741.994477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763741.994481:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763741.994487:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.994498:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763741.994506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763741.994511:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763741.994522:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763741.994533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020daf100 x1876702067707904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763741.994542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067708416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763741.994574:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067707904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763741.994583:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763741.994594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763741.994597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763741.994604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763741.994608:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763741.994614:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067707904:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763741.994625:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1209 00000100:00080000:0.0:1789763741.994636:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763741.994674:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763741.994684:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763741.994686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763741.994713:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067708544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763741.994730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763741.994753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020dac700 x1876702067708032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763741.994772:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067708032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763741.994783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.994787:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.994791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763741.994843:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763741.994870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763741.994885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020dad880 x1876702067708160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763741.994900:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067708160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763741.994910:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.994914:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.994918:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763741.994925:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763741.994945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763741.994957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020daf480 x1876702067708288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763741.994971:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067708288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763741.994989:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.994993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.994997:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763741.995003:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067708416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763741.995022:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763741.995035:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020dac380 x1876702067708416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763741.995047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067708416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763741.995056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.995060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.995063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067708416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763741.995083:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067708544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763741.995103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763741.995115:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763741/real 1789763741] req@ffff936020dae680 x1876702067708544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763741.995129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067708544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763741.995138:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763741.995141:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763741.995146:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067708544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763742.762151:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763743.786104:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763744.810008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763745.834073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763746.857993:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763747.113902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.113917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763747.113922:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763747.113936:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.113949:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.113959:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763747.114006:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067708672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763747.114044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.114051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763747.114055:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763747.114062:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.114077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763747.114093:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763747.114125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067708672:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763747.114139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067708800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763747.114172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.114178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763747.114181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763747.114189:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114214:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.114221:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763747.114228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789763747.114229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763747.114259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067708928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763747.114276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.114283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763747.114286:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763747.114289:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020daed80 x1876702067708672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763747.114293:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114299:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.114306:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763747.114314:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763747.114320:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067708672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763747.114333:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067709056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763747.114340:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.114348:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763747.114351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763747.114356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067708672:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763747.114357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763747.114361:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763747.114366:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114373:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.114379:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763747.114386:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763747.114394:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763747.114401:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067709184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763747.114419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.114425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763747.114428:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763747.114434:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763747.114435:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114441:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763747.114447:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763747.114456:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763747.114458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020dac000 x1876702067708800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763747.114473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067709312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763747.114483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067708800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763747.114491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763747.114498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763747.114500:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763747.114502:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763747.114506:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763747.114508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1214 00000100:00080000:0.0:1789763747.114514:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763747.114514:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708800:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763747.114521:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763747.114526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708928:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789763747.114529:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763747.114545:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067709440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763747.114559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763747.114579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020dafb80 x1876702067708928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763747.114629:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067708928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763747.114649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.114657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763747.114664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067708928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763747.114677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763747.114713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763747.114729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020dae680 x1876702067709056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763747.114744:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067709056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763747.114755:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.114759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763747.114763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763747.114782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763747.114848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763747.114864:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020dac380 x1876702067709184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763747.114880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067709184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763747.114890:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.114894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763747.114898:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763747.114904:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067709312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763747.114927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763747.114940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020daf480 x1876702067709312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763747.114954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067709312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763747.114964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.114967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763747.114970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067709312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763747.114976:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067709440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763747.114995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763747.115007:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763747/real 1789763747] req@ffff936020dad880 x1876702067709440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763747.115021:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067709440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763747.115038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763747.115042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763747.115045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067709440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763747.882027:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763748.906065:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763749.930123:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763750.954003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763751.978032:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763752.234049:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763752.234070:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763752.234085:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234102:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763752.234113:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763752.234176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067709568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763752.234219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763752.234227:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763752.234232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234237:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763752.234243:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763752.234257:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763752.234275:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067709696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763752.234291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763752.234300:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763752.234308:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067709568:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763752.234312:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234323:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763752.234331:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763752.234339:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763752.234357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067709824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763752.234371:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763752.234379:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763752.234384:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234391:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763752.234396:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763752.234417:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763752.234431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763752.234443:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067709952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763752.234460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234465:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763752.234467:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763752.234484:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234489:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763752.234493:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763752.234501:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763752.234514:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067710080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763752.234522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020dac700 x1876702067709568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763752.234541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763752.234593:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763752.234595:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067709568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763752.234611:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234624:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763752.234630:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763752.234639:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763752.234642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.234655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067709568:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763752.234663:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763752.234685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763752.234701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067710208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763752.234718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763752.234722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763752.234728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763752.234732:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763752.234736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020daf100 x1876702067709696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763752.234744:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1220 00000100:00080000:0.0:1789763752.234763:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763752.234775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067709696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.234792:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763752.234864:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763752.234882:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000008:00100000:0.0:1789763752.234883:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763752.234890:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763752.234904:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763752.234920:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067710336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763752.234932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763752.234956:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020dad500 x1876702067709824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763752.234985:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067709824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.234992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763752.234996:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.235002:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763752.235010:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763752.235027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763752.235034:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020dadf80 x1876702067709952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763752.235042:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067709952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.235047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763752.235049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.235051:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067709952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763752.235056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763752.235067:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763752.235079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020daf800 x1876702067710080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763752.235086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067710080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.235092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763752.235094:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.235096:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763752.235109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067710208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763752.235122:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763752.235129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020dae300 x1876702067710208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763752.235136:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067710208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.235142:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763752.235144:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.235146:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067710208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763752.235149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067710336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763752.235160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763752.235173:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763752/real 1789763752] req@ffff936020dad180 x1876702067710336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763752.235181:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067710336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763752.235186:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763752.235187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763752.235189:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067710336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763753.002139:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763754.026073:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763755.050111:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763756.074042:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763757.098004:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763757.353971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763757.353991:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763757.353999:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763757.354016:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354034:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354047:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763757.354097:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067710464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763757.354136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763757.354141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763757.354144:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763757.354148:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354153:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354157:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763757.354169:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763757.354185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067710592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763757.354201:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763757.354206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763757.354209:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763757.354218:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067710464:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763757.354221:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354245:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354251:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763757.354263:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763757.354282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067710720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763757.354296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763757.354301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763757.354304:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763757.354308:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354315:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354319:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763757.354325:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763757.354339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763757.354342:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067710848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763757.354358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763757.354362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763757.354365:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763757.354371:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354377:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354381:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763757.354387:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763757.354402:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067710976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763757.354414:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763757.354423:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa6300 x1876702067710464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763757.354424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763757.354430:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763757.354443:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354454:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354465:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763757.354472:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763757.354480:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067710464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763757.354493:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067711104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763757.354506:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763757.354515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763757.354519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763757.354551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763757.354559:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763757.354561:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067710464:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763757.354578:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1225 00000100:00080000:3.0:1789763757.354590:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763757.354599:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763757.354610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710592:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763757.354612:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763757.354650:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067711232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763757.354667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.354701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa7800 x1876702067710592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.354724:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067710592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.354737:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.354742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.354747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763757.354755:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763757.354777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.354791:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa7b80 x1876702067710720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.354853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067710720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.354864:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.354868:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.354872:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763757.354880:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763757.354906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.354920:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa5180 x1876702067710848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.354935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067710848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.354978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.354982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.354986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763757.354993:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763757.355019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.355041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa4380 x1876702067710976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.355064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067710976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.355081:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.355089:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.355095:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067710976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763757.355108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067711104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763757.355146:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.355163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa4a80 x1876702067711104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.355178:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067711104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.355188:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.355191:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.355196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067711104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763757.355225:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067711232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763757.355257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763757.355280:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763757/real 1789763757] req@ffff936010aa5880 x1876702067711232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763757.355305:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067711232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763757.355322:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763757.355328:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763757.355335:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067711232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763758.122176:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763759.145957:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763760.170041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763761.193934:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763762.217959:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763762.473886:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.473898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763762.473902:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.473914:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.473923:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763762.473931:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763762.473964:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067711360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763762.473985:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.473988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763762.473990:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.473994:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.473998:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763762.474001:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474011:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763762.474016:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067711360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763762.474026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067711488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763762.474037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.474040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763762.474042:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.474047:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.474050:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763762.474054:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474058:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763762.474068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067711616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763762.474077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.474080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763762.474082:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.474086:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.474090:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763762.474093:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474098:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763762.474106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067711744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763762.474113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.474116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763762.474118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.474122:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.474126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763762.474132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763762.474133:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474142:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763762.474151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067711872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763762.474158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.474161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763762.474163:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763762.474166:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.474169:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763762.474172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474176:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763762.474193:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa5f80 x1876702067711360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067711360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474221:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474232:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067711360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763762.474257:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067712000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763762.474265:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711488:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763762.474271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763762.474276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763762.474279:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763762.474309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763762.474323:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa6680 x1876702067711488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474339:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067711488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474355:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474360:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763762.474369:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711616:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763762.474386:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1230 00000100:00080000:3.0:1789763762.474392:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763762.474398:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763762.474399:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763762.474407:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763762.474417:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa7100 x1876702067711616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067711616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763762.474454:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763762.474470:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067712128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763762.474481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763762.474511:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa4000 x1876702067711744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067711744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474534:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474536:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763762.474543:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763762.474558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763762.474566:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa5500 x1876702067711872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474574:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067711872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474580:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474583:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067711872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763762.474600:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763762.474613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763762.474621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa4700 x1876702067712000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474629:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067712000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763762.474649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763762.474660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763762.474667:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763762/real 1789763762] req@ffff936010aa6a00 x1876702067712128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763762.474675:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067712128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763762.474681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763762.474683:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763762.474685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763763.242075:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763764.266022:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763765.290014:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763766.313995:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763767.337925:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763767.593902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.593915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763767.593918:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.593928:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.593954:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.593960:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763767.593995:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067712256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763767.594021:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594024:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.594026:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594028:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594036:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763767.594045:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067712384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763767.594076:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594081:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00100000:1.0:1789763767.594084:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067712256:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763767.594084:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594089:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594098:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763767.594107:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067712512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763767.594118:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594121:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.594124:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594127:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594131:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763767.594139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067712640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763767.594150:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594153:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.594156:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594161:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789763767.594163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763767.594178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067712768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763767.594188:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.594194:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594198:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763767.594206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa4e00 x1876702067712256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763767.594213:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067712896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763767.594225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763767.594227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763767.594228:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763767.594231:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1235 00000100:00080000:3.0:1789763767.594233:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763767.594235:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763767.594257:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763767.594260:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067712256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763767.594267:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067713024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763767.594298:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594310:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067712256:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763767.594334:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763767.594355:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594365:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa7480 x1876702067712384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594374:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067712384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594380:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594383:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594385:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763767.594389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763767.594409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa5c00 x1876702067712512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067712512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594446:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594450:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763767.594474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763767.594493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa6a00 x1876702067712640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067712640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594532:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763767.594535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763767.594546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa4700 x1876702067712768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067712768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594570:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067712768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763767.594573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763767.594584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594591:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa5500 x1876702067712896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594599:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067712896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594608:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067712896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763767.594612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763767.594623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763767.594629:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763767/real 1789763767] req@ffff936010aa4000 x1876702067713024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763767.594637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067713024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763767.594642:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763767.594644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763767.594646:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789763768.362123:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763769.386008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763770.409978:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763771.433961:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763772.457993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763772.713910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.713923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763772.713928:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.713940:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.713952:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763772.713961:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763772.714014:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067713152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763772.714045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763772.714052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714056:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714060:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763772.714064:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714074:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763772.714088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067713280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763772.714100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763772.714107:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714115:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763772.714119:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714125:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763772.714134:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067713152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763772.714143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067713408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763772.714158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763772.714164:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714168:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714171:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763772.714175:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714181:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763772.714195:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067713536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763772.714206:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763772.714213:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714216:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714220:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763772.714227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763772.714235:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714245:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763772.714268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067713664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763772.714285:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa7100 x1876702067713152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763772.714289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763772.714312:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714328:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714341:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763772.714347:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067713152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763772.714356:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763772.714374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714383:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714389:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067713152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763772.714395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067713792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763772.714406:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713280:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763772.714437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763772.714443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763772.714446:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763772.714453:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1240 00000100:00080000:3.0:1789763772.714467:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763772.714475:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763772.714475:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763772.714488:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763772.714503:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa6680 x1876702067713280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763772.714511:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067713920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763772.714549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067713280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.714562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763772.714607:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763772.714643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763772.714656:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa5f80 x1876702067713408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763772.714669:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067713408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.714684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763772.714697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763772.714716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763772.714735:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa5880 x1876702067713536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763772.714747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067713536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.714756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763772.714768:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763772.714787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763772.714837:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa4a80 x1876702067713664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763772.714851:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067713664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.714860:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714868:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067713664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763772.714874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763772.714905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763772.714938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa4380 x1876702067713792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763772.714966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067713792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.714981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.714987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.714991:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763772.714999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763772.715026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763772.715043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763772/real 1789763772] req@ffff936010aa5180 x1876702067713920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763772.715058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067713920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763772.715066:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763772.715070:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763772.715073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067713920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763773.481962:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763774.505916:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763775.529948:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763776.554103:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763777.578104:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763777.833956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.833965:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763777.833970:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.833981:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.833994:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834002:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763777.834047:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067714048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763777.834073:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834076:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763777.834078:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834081:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834083:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834085:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834094:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763777.834104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067714176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763777.834115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763777.834120:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834125:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834129:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834134:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763777.834153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067714304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763777.834161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067714048:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763777.834176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834179:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763777.834180:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834184:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834188:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834191:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763777.834207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067714432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763777.834214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763777.834218:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834220:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834231:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834234:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834237:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763777.834242:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763777.834248:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067714560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763777.834256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763777.834260:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834263:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834267:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834269:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834272:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763777.834280:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067714688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763777.834286:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763777.834290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763777.834291:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020dadc00 x1876702067714048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763777.834292:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763777.834301:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1245 00000100:00080000:0.0:1789763777.834307:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763777.834311:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763777.834315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763777.834319:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067714048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763777.834324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067714816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763777.834337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834347:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067714048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763777.834354:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763777.834372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020dace00 x1876702067714176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067714176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834412:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834414:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834417:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763777.834438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763777.834450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020daea00 x1876702067714304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067714304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763777.834479:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763777.834491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834497:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020dad180 x1876702067714432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834505:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067714432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834511:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834513:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763777.834516:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763777.834526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020dae300 x1876702067714560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834539:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067714560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834554:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067714560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763777.834557:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067714688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763777.834568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020daf800 x1876702067714688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067714688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834585:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067714688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763777.834593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067714816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763777.834603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763777.834609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763777/real 1789763777] req@ffff936020dadf80 x1876702067714816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763777.834615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067714816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763777.834621:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763777.834623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763777.834625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067714816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763778.601970:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763779.625984:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763780.649919:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763781.673989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763782.697961:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763782.953891:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.953901:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763782.953904:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763782.953939:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:3.0:1789763782.953951:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.953958:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763782.953998:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067714944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763782.954022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763782.954026:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763782.954029:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:3.0:1789763782.954032:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.954035:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954043:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763782.954082:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067715072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763782.954092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763782.954096:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763782.954101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00100000:1.0:1789763782.954102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067714944:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763782.954104:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.954110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954115:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763782.954125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067715200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763782.954132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763782.954135:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763782.954138:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:3.0:1789763782.954140:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.954144:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954147:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763782.954155:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067715328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763782.954161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763782.954165:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763782.954169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763782.954172:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:3.0:1789763782.954176:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.954179:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954182:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763782.954189:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067715456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763782.954195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763782.954199:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763782.954202:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:3.0:1789763782.954204:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763782.954207:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954210:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763782.954211:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa7b80 x1876702067714944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763782.954221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067715584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763782.954230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067714944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763782.954240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763782.954246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763782.954248:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763782.954249:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763782.954252:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1250 00000100:00080000:1.0:1789763782.954256:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763782.954257:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763782.954263:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067714944:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763782.954264:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763782.954271:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763782.954279:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067715712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763782.954291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763782.954310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa7800 x1876702067715072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954327:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067715072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954332:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954335:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954337:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763782.954359:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763782.954374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa6300 x1876702067715200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954389:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067715200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954395:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954399:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763782.954402:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763782.954413:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa5180 x1876702067715328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954426:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067715328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954431:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954433:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954435:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763782.954438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763782.954448:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa4380 x1876702067715456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067715456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954466:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763782.954472:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067715584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763782.954483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa4a80 x1876702067715584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067715584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067715584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763782.954508:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067715712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763782.954518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763782.954523:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763782/real 1789763782] req@ffff936010aa5880 x1876702067715712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763782.954530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067715712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763782.954539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763782.954541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763782.954543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067715712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763783.721921:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763784.745954:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763785.769937:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763786.793949:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763787.817962:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763788.073888:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.073902:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763788.073906:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.073917:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.073927:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.073933:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763788.073965:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067715840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763788.073989:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.073992:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763788.073993:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.073996:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.073998:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.074001:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074010:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763788.074018:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067715968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763788.074026:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067715840:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763788.074041:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.074043:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763788.074044:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.074048:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.074051:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.074053:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074056:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763788.074065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067716096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763788.074071:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.074074:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763788.074075:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.074078:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.074080:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.074082:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074086:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763788.074089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763788.074096:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067716224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763788.074105:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.074107:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763788.074108:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.074111:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.074114:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.074116:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074120:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763788.074127:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef7800 x1876702067715840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763788.074129:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067716352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763788.074154:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763788.074161:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763788.074164:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.074168:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.074171:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763788.074174:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074177:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763788.074178:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067715840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763788.074186:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067716480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763788.074189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763788.074201:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763788.074203:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067715840:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763788.074211:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763788.074215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763788.074221:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1255 00000100:00080000:1.0:1789763788.074225:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763788.074229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715968:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763788.074232:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763788.074241:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763788.074253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763788.074258:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067716608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763788.074270:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef7b80 x1876702067715968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067715968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074318:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074322:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074326:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067715968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763788.074333:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763788.074358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763788.074372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef7480 x1876702067716096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067716096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074404:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074408:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763788.074414:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763788.074439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763788.074453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef6d80 x1876702067716224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067716224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074479:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763788.074490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763788.074509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763788.074521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef6300 x1876702067716352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067716352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074544:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763788.074548:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067716480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763788.074563:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763788.074570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef4e00 x1876702067716480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074577:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067716480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074586:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067716480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763788.074608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067716608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763788.074622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763788.074629:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763788/real 1789763788] req@ffff936010ef5500 x1876702067716608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763788.074637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067716608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763788.074643:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763788.074645:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763788.074647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067716608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763788.841995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763789.866096:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763790.889998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763791.914014:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763792.937999:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763793.193928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.193937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763793.193941:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.193951:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.193960:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.193966:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763793.193998:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067716736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763793.194022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763793.194026:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194029:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194032:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194034:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194042:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763793.194051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067716864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763793.194059:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763793.194064:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194067:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194072:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194075:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763793.194081:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067716992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763793.194092:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067716736:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763793.194094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763793.194113:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194116:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194118:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194120:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194123:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763793.194136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067717120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763793.194147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763793.194151:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194153:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194156:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194158:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194161:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763793.194170:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067717248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763793.194175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763793.194179:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194181:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194186:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194188:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763793.194196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763793.194198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067717376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763793.194207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763793.194209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763793.194211:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763793.194215:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1261 00000100:00080000:3.0:1789763793.194221:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763793.194224:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763793.194229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763793.194235:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067717504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763793.194283:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa5f80 x1876702067716736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194308:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067716736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194322:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067716736:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763793.194350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763793.194378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa6680 x1876702067716864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067716864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194423:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763793.194456:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763793.194474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa7100 x1876702067716992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067716992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194513:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194516:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194520:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067716992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763793.194527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763793.194545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194555:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa4000 x1876702067717120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194563:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067717120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194568:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194571:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763793.194576:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763793.194590:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa5500 x1876702067717248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067717248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194609:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194611:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194614:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763793.194617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067717376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763793.194629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa4700 x1876702067717376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067717376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194647:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067717376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763793.194654:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067717504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763793.194665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763793.194671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763793/real 1789763793] req@ffff936010aa6a00 x1876702067717504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763793.194678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067717504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763793.194682:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763793.194684:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763793.194687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067717504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763793.962017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763794.986054:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763796.009973:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763797.033964:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763798.057974:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763798.313860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.313869:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763798.313873:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.313883:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.313892:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.313899:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763798.313934:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067717632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763798.313965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.313970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763798.313972:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.313977:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.313982:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.313987:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.313998:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763798.314018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067717760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763798.314025:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067717632:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763798.314029:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.314034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763798.314035:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.314039:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.314043:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.314046:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.314050:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763798.314063:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067717888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763798.314071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.314074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763798.314077:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.314080:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.314083:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.314086:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.314089:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763798.314098:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067718016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763798.314106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.314110:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763798.314114:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.314121:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.314124:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.314131:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.314137:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763798.314148:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763798.314151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067718144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763798.314162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.314165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763798.314166:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.314169:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.314174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763798.314177:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.314183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763798.314187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763798.314207:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763798.314225:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763798.314278:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067718272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763798.314295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763798.314299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763798.314301:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763798.314304:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1266 00000100:00080000:3.0:1789763798.314310:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763798.314317:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067718272:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763798.315368:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763798.315379:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763798.315402:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067718400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763798.315457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067718400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763799.081986:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763800.105961:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763801.130039:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763801.386149:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763801.386196:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763801.386232:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763801.386297:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763801.386311:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763801.386324:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763801.386336:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763801.386348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa7480 x1876702067717760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763801.386348:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763801.386362:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789763801.386377:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067717760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386399:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386402:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763801.386408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa4e00 x1876702067717888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067717888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386426:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386429:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067717888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763801.386433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa6a00 x1876702067718016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386440:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067718016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386447:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386451:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386455:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763801.386462:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa5c00 x1876702067717632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386482:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067717632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386494:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386498:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067717632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763801.386535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa4000 x1876702067718400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386550:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067718400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067718400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763801.386574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa5500 x1876702067718272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386602:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067718272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386611:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386617:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067718272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763801.386625:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763798/real 1789763801] req@ffff936010aa4700 x1876702067718144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763801.386640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067718144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763801.386649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763801.386653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763801.386657:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718144:192.168.203.137@tcp:8:kworker.0 00010000:00010000:1.0:1789763802.154111:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763803.177960:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763803.433934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.433948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763803.433953:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.433966:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.433976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763803.433984:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763803.434021:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067718528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763803.434054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434059:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763803.434061:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.434069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763803.434073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434082:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763803.434094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067718656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763803.434106:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763803.434112:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434116:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.434122:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763803.434123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067718528:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763803.434128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763803.434149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067718784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763803.434165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763803.434171:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434174:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.434179:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763803.434184:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434204:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763803.434205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763803.434221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067718912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763803.434232:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434236:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763803.434238:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434242:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.434246:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763803.434251:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434256:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763803.434264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa7100 x1876702067718528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763803.434269:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067719040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763803.434295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763803.434301:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434306:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00100000:1.0:1789763803.434311:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067718528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763803.434313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763803.434323:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434331:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763803.434335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434342:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763803.434347:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067719168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763803.434351:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067718528:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763803.434372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763803.434376:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763803.434378:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763803.434383:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1271 00000100:00080000:3.0:1789763803.434387:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763803.434395:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718656:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763803.434395:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763803.434404:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763803.434421:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067719296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763803.434437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434455:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa6680 x1876702067718656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434470:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067718656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434491:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763803.434498:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763803.434521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa5f80 x1876702067718784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067718784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434570:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763803.434579:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763803.434602:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa5880 x1876702067718912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434627:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067718912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434642:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067718912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763803.434649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763803.434671:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434683:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa4a80 x1876702067719040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434696:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067719040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434707:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763803.434717:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067719168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763803.434739:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa4380 x1876702067719168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067719168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434774:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434781:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067719168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763803.434847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067719296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763803.434871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763803.434884:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763803/real 1789763803] req@ffff936010aa5180 x1876702067719296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763803.434902:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067719296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763803.434910:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763803.434913:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763803.434916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067719296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763804.202001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763805.226014:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763806.249970:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763807.273970:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763808.297969:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763808.553863:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.553874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763808.553879:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.553889:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.553900:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763808.553907:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763808.553944:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067719424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763808.553970:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.553975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763808.553977:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.553981:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.553985:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763808.553990:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763808.554000:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763808.554009:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067719424:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763808.554016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067719552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763808.554030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.554034:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763808.554036:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.554040:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.554045:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763808.554049:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763808.554054:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763808.554068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763808.554069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067719680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763808.554082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.554086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763808.554088:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.554093:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.554098:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763808.554103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa6300 x1876702067719424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763808.554104:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763808.554114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763808.554122:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067719424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763808.554130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067719808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763808.554133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763808.554141:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067719424:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763808.554150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.554172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763808.554174:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.554179:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.554184:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763808.554188:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554193:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719552:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763808.554194:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763808.554212:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067719936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763808.554221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763808.554231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763808.554237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa7800 x1876702067719552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763808.554237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763808.554249:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067719552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763808.554249:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763808.554258:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:1.0:1789763808.554263:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763808.554265:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763808.554268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554274:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719552:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763808.554274:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763808.554284:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763808.554285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763808.554302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067720064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763808.554306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554317:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa7b80 x1876702067719680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763808.554319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763808.554328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763808.554332:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763808.554337:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067719680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763808.554340:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1276 00000100:00080000:3.0:1789763808.554347:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763808.554349:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763808.554354:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789763808.554356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554362:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719680:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763808.554363:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763808.554368:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763808.554378:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067720192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763808.554386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa5180 x1876702067719808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763808.554406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067719808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763808.554416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763808.554419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554421:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763808.554433:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789763808.554444:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554451:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa4380 x1876702067719936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763808.554458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067719936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763808.554463:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763808.554465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067719936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789763808.554470:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763808.554480:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa4a80 x1876702067720064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763808.554493:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067720064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763808.554498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763808.554500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554502:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763808.554506:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789763808.554516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763808.554522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763808/real 1789763808] req@ffff936010aa5880 x1876702067720192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763808.554528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067720192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763808.554533:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763808.554535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789763808.554536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763809.321978:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763810.346046:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763811.369930:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763812.393958:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763813.417944:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763813.673897:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.673906:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763813.673910:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.673918:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.673927:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.673932:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763813.673963:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067720320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.673989:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.673992:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763813.673993:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.673996:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.673998:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.674001:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763813.674008:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763813.674015:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067720448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.674023:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.674026:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763813.674028:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.674030:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.674032:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.674036:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763813.674041:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763813.674041:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067720320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763813.674052:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067720576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.674060:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.674062:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763813.674064:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.674066:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.674068:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.674070:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763813.674074:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763813.674083:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067720704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.674090:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.674092:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763813.674093:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.674095:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.674104:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.674106:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763813.674109:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763813.674126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763813.674135:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067720832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.674143:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.674146:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763813.674147:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763813.674150:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:1.0:1789763813.674153:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763813.674156:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763813.674160:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763813.674170:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef7100 x1876702067720320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763813.674171:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067720960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763813.674198:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763813.674202:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763813.674209:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763813.674209:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067720320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763813.674220:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1281 00000100:00080000:2.0:1789763813.674224:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763813.674226:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763813.674230:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763813.674235:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674236:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067720320:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789763813.674246:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763813.674247:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763813.674266:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067721088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763813.674275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674288:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef6680 x1876702067720448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674299:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067720448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674305:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674308:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674310:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763813.674329:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763813.674341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef5f80 x1876702067720576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674355:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067720576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674361:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763813.674368:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763813.674379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674385:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef6a00 x1876702067720704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674392:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067720704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763813.674404:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763813.674415:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef5500 x1876702067720832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067720832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674434:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674436:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067720832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763813.674442:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763813.674453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674459:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef4e00 x1876702067720960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067720960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067720960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763813.674478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763813.674488:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763813.674494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763813/real 1789763813] req@ffff936010ef6300 x1876702067721088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763813.674505:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067721088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763813.674511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763813.674512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763813.674514:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763814.442052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763815.466079:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763816.490080:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763817.514050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763818.538003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763818.794001:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794022:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763818.794045:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.794061:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794107:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763818.794120:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763818.794212:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067721216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763818.794258:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794265:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763818.794268:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.794276:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794282:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763818.794292:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763818.794309:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763818.794336:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067721344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763818.794364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067721216:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763818.794365:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794389:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763818.794397:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.794408:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794415:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763818.794424:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763818.794434:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763818.794465:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067721472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763818.794483:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794490:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763818.794494:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.794513:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794520:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763818.794528:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763818.794543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:1.0:1789763818.794549:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763818.794579:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067721600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763818.794618:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794627:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763818.794633:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.794643:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794653:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763818.794665:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef6d80 x1876702067721216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763818.794675:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763818.794717:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763818.794737:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067721216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763818.794752:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067721728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763818.794778:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763818.794793:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.794805:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763818.794823:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763818.794892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763818.794896:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1286 00000100:00080000:1.0:1789763818.794913:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763818.794917:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067721216:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763818.794933:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763818.794951:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763818.794980:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067721856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763818.794985:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721344:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763818.795042:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763818.795048:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763818.795052:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763818.795066:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1286 00000800:00000100:2.0:1789763818.795072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763818.795105:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763818.795121:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763818.795133:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763818.795138:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef7480 x1876702067721344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763818.795168:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067721984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763818.795178:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067721344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795277:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763818.795292:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763818.795331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763818.795351:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef7b80 x1876702067721472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763818.795375:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067721472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795407:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763818.795419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763818.795451:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763818.795469:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef7800 x1876702067721600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763818.795508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067721600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795527:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763818.795547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763818.795595:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763818.795616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef4380 x1876702067721728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763818.795641:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067721728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067721728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763818.795704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763818.795747:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763818.795772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef4000 x1876702067721856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763818.795796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067721856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795847:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795855:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795860:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763818.795869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763818.795899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763818.795927:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763818/real 1789763818] req@ffff936010ef5c00 x1876702067721984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763818.795942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067721984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763818.795952:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763818.795955:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763818.795959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067721984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763819.562150:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763820.585973:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763821.609982:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763822.634079:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763823.658025:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763823.913923:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.913932:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763823.913936:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.913946:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.913958:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.913966:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763823.914010:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067722112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763823.914034:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914037:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763823.914051:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.914054:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.914057:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.914061:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763823.914071:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763823.914081:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067722240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763823.914092:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067722112:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763823.914098:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763823.914103:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.914108:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.914110:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.914112:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763823.914116:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763823.914131:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067722368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763823.914138:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763823.914143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.914145:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.914148:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.914151:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763823.914155:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763823.914163:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067722496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763823.914169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763823.914175:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763823.914180:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.914183:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.914187:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.914191:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763823.914194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763823.914203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067722624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763823.914212:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914216:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763823.914221:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763823.914221:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef5880 x1876702067722112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763823.914231:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00080000:1.0:1789763823.914239:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763823.914246:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067722112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763823.914254:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789763823.914258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789763823.914260:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763823.914264:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067722112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763823.914276:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067722752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763823.914285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763823.914287:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763823.914289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763823.914295:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1291 00000100:00100000:2.0:1789763823.914295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722240:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763823.914302:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763823.914305:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763823.914309:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763823.914320:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067722880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763823.914326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914340:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef4a80 x1876702067722240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067722240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914367:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763823.914374:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763823.914386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef5180 x1876702067722368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067722368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763823.914413:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763823.914425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef4700 x1876702067722496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914448:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067722496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763823.914471:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763823.914492:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914505:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef5c00 x1876702067722624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067722624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914524:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067722624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763823.914530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067722752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763823.914553:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914561:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef4000 x1876702067722752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914568:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067722752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914576:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914578:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067722752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763823.914591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067722880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763823.914603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763823.914610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763823/real 1789763823] req@ffff936010ef4380 x1876702067722880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763823.914617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067722880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763823.914622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763823.914624:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763823.914625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067722880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763824.681967:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763825.705983:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763826.729957:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763827.753930:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763828.778098:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763829.033973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.033988:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763829.033995:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034030:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034049:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034062:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763829.034124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067723008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763829.034161:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034165:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034169:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763829.034185:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763829.034198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067723136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034218:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763829.034220:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034224:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034226:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034233:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067723008:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763829.034240:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763829.034258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067723264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763829.034281:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034287:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034313:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034319:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763829.034325:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763829.034332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763829.034341:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067723392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763829.034370:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034374:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034378:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034383:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763829.034388:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763829.034406:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa5f80 x1876702067723008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763829.034408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067723520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763829.034459:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763829.034460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067723008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763829.034469:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00080000:3.0:1789763829.034478:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763829.034485:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763829.034485:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789763829.034496:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789763829.034497:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763829.034504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067723008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763829.034517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763829.034519:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067723648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763829.034536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763829.034542:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:2.0:1789763829.034546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763829.034548:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763829.034559:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1296 00000100:00100000:2.0:1789763829.034565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa6680 x1876702067723136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763829.034568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763829.034581:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034587:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067723136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763829.034591:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763829.034607:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763829.034613:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067723776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763829.034614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763829.034683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763829.034702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034711:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa7100 x1876702067723264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763829.034720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067723264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763829.034726:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763829.034728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763829.034734:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763829.034749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034757:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa4700 x1876702067723392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763829.034766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067723392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763829.034772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763829.034774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763829.034780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763829.034794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034802:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa5500 x1876702067723520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763829.034811:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067723520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763829.034846:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763829.034849:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034852:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067723520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763829.034857:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067723648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763829.034873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034883:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa4000 x1876702067723648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763829.034893:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067723648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763829.034899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763829.034902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034904:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067723648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763829.034909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067723776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763829.034922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763829.034947:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763829/real 1789763829] req@ffff936010aa5c00 x1876702067723776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763829.034957:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067723776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763829.034964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763829.034966:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763829.034968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067723776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763829.801980:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763830.826109:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763831.850010:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763832.874048:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763833.898128:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763834.154017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763834.154037:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763834.154053:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154071:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.154084:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763834.154145:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067723904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.154185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763834.154195:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763834.154200:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154205:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.154209:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763834.154223:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763834.154245:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067724032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.154259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763834.154267:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763834.154273:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154277:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.154281:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763834.154289:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763834.154306:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067724160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.154319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763834.154328:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763834.154333:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154338:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.154345:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763834.154351:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763834.154368:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067723904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763834.154374:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067724288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.154395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763834.154402:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763834.154407:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154412:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.154417:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763834.154424:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763834.154440:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067724416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.154451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.154456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1789763834.154482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.154550:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa6a00 x1876702067723904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.154582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067723904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763834.154590:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763834.154616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.154624:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.154629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067723904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763834.154638:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763834.154673:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.154694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa4e00 x1876702067724032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.154722:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067724032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763834.154753:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.154762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.154772:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724032:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763834.154775:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.154789:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763834.154850:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724160:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763834.154905:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763834.154912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763834.154921:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763834.154943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa7480 x1876702067724160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763834.154952:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067724544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763834.155005:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763834.155011:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763834.155014:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763834.155022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067724160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763834.155027:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1301 00000100:00080000:3.0:1789763834.155039:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763834.155052:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763834.155064:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763834.155067:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.155084:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789763834.155086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067724672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763834.155103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763834.155177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763834.155228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.155250:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa5c00 x1876702067724288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.155267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067724288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763834.155278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.155282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.155287:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763834.155295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763834.155321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.155334:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa4000 x1876702067724416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.155348:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067724416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763834.155358:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.155361:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.155365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763834.155371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067724544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763834.155392:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.155404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa5500 x1876702067724544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.155419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067724544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763834.155428:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.155431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.155435:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067724544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763834.155441:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067724672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763834.155461:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763834.155474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763834/real 1789763834] req@ffff936010aa4700 x1876702067724672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763834.155488:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067724672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763834.155497:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763834.155500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763834.155504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067724672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763834.921969:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763835.946022:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763836.970151:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763837.994098:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763839.018051:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763839.274024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763839.274040:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274052:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274063:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274070:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763839.274108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067724800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763839.274145:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274148:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274151:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274155:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763839.274196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067724928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763839.274213:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274219:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274224:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274229:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274235:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763839.274249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067725056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274279:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763839.274281:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274284:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274287:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274297:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763839.274308:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067724800:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763839.274330:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067725184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763839.274351:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274356:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274359:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274362:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274365:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763839.274377:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067725312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763839.274392:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763839.274395:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274398:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274401:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274405:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763839.274414:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067725440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763839.274422:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763839.274425:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763839.274427:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763839.274437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763839.274445:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1307 00000100:00080000:1.0:1789763839.274452:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763839.274457:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763839.274463:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763839.274479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067725568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763839.274536:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef7800 x1876702067724800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.274566:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067724800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.274583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.274589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.274594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067724800:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763839.274616:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763839.274648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.274666:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef7b80 x1876702067724928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.274703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067724928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.274724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.274728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.274744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067724928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763839.274781:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763839.274845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.274861:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef7480 x1876702067725056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.274886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067725056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.274897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.274902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.274906:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763839.274915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763839.274978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.274994:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef6d80 x1876702067725184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.275010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067725184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.275032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.275036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.275041:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763839.275049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763839.275072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.275085:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef6300 x1876702067725312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.275099:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067725312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.275109:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.275112:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.275116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763839.275123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067725440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763839.275259:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.275285:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef4e00 x1876702067725440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.275312:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067725440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.275323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.275328:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.275332:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067725440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763839.275340:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067725568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763839.275370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763839.275385:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763839/real 1789763839] req@ffff936010ef5500 x1876702067725568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763839.275400:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067725568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763839.275410:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763839.275413:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763839.275418:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067725568:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763840.042037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763841.066115:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763842.090099:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763843.114113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763844.138071:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763844.393998:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763844.394017:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763844.394027:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394046:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394068:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394080:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763844.394142:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067725696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763844.394200:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763844.394209:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763844.394213:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394221:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394229:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394240:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394259:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763844.394290:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067725824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763844.394316:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763844.394318:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067725696:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763844.394335:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763844.394342:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394349:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394356:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394362:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763844.394394:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067725952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763844.394407:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763844.394412:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763844.394415:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394420:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394426:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394433:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763844.394445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394450:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763844.394471:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067726080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763844.394485:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763844.394490:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763844.394492:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394502:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394507:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394512:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394517:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763844.394527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061dce00 x1876702067725696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763844.394544:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067726208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763844.394630:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763844.394643:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763844.394646:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763844.394666:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00100000:0.0:1789763844.394669:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067725696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763844.394673:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394686:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394699:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763844.394730:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067726336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763844.394736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763844.394757:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763844.394761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763844.394770:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763844.394774:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763844.394778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067725696:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763844.394791:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1312 00000100:00080000:2.0:1789763844.394868:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763844.394876:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763844.394884:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763844.394943:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763844.394954:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067726464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763844.395025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061dd180 x1876702067725824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067725824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395123:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395133:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763844.395143:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763844.395175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395190:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061dc000 x1876702067725952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395224:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067725952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395234:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395241:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067725952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763844.395249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763844.395271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061de680 x1876702067726080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395297:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067726080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395306:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395310:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763844.395319:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763844.395337:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395349:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061dfb80 x1876702067726208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395361:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067726208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763844.395383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067726336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763844.395401:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395412:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061df800 x1876702067726336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395424:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067726336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395434:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395437:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395441:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067726336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763844.395464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067726464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763844.395485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763844.395496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763844/real 1789763844] req@ffff9360061dca80 x1876702067726464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763844.395509:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067726464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763844.395518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763844.395522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763844.395525:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067726464:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763845.162082:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763846.186084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763847.210011:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763848.234097:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763849.258078:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763849.513992:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514003:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763849.514007:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514017:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514026:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514032:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763849.514066:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067726592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763849.514096:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514099:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763849.514100:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514104:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514106:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514108:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763849.514115:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763849.514123:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067726720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763849.514131:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763849.514136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514139:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514141:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514144:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763849.514148:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763849.514174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067726592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763849.514183:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067726848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763849.514206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514213:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763849.514228:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514236:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514244:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514267:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763849.514273:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763849.514286:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067726976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763849.514294:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763849.514300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763849.514301:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763849.514304:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514309:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514313:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514316:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763849.514320:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763849.514331:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067727104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763849.514342:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514346:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763849.514348:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514355:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00100000:0.0:1789763849.514356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa7100 x1876702067726592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763849.514362:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514380:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067726592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763849.514387:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763849.514401:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763849.514405:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067727232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763849.514408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514416:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067726592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763849.514430:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726720:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763849.514433:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763849.514437:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763849.514438:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763849.514442:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1317 00000100:00080000:3.0:1789763849.514446:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763849.514450:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763849.514454:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763849.514458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763849.514468:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067727360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763849.514474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa6680 x1876702067726720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514492:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067726720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514502:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763849.514549:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763849.514574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514584:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa5f80 x1876702067726848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067726848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763849.514607:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763849.514620:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa5880 x1876702067726976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067726976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514643:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067726976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763849.514648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763849.514659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514665:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa4a80 x1876702067727104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067727104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514692:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763849.514695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067727232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763849.514707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa4380 x1876702067727232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514721:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067727232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514729:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514732:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067727232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763849.514735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067727360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763849.514745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763849.514752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763849/real 1789763849] req@ffff936010aa5180 x1876702067727360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763849.514759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067727360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763849.514764:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763849.514766:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763849.514768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067727360:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763850.282151:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763851.306227:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763852.330051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763853.354110:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763854.378129:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763854.633964:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.633980:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763854.633987:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634002:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634019:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634030:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763854.634089:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067727488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763854.634133:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634138:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763854.634140:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634144:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634150:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634157:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763854.634175:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763854.634198:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067727616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763854.634217:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067727488:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763854.634246:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634253:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763854.634257:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634264:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634269:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634275:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763854.634283:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763854.634300:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067727744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763854.634312:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634316:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763854.634318:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634322:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634325:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634332:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763854.634339:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763854.634340:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763854.634357:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067727872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763854.634368:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634371:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763854.634374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634379:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634384:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634388:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763854.634393:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763854.634402:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067728000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763854.634411:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634414:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763854.634416:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634420:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634424:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634428:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763854.634434:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763854.634443:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa7b80 x1876702067727488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763854.634450:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067728128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763854.634490:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763854.634495:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763854.634498:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763854.634503:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1322 00000100:00080000:3.0:1789763854.634508:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763854.634513:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763854.634520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067727488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763854.634521:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763854.634540:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067728256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763854.634587:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.634598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.634609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067727488:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763854.634653:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763854.634714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.634741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa7800 x1876702067727616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.634771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067727616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.634790:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.634797:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.634804:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763854.634912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763854.634954:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.634975:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa6300 x1876702067727744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.634996:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067727744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.635005:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.635009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.635013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763854.635020:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763854.635045:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.635057:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa5180 x1876702067727872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.635070:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067727872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.635078:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.635082:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.635085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067727872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763854.635090:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763854.635107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.635118:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa4380 x1876702067728000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.635128:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067728000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.635135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.635139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.635142:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763854.635148:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067728128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763854.635165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.635174:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa4a80 x1876702067728128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.635185:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067728128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.635199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.635205:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.635210:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067728128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763854.635221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067728256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763854.635255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763854.635275:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763854/real 1789763854] req@ffff936010aa5880 x1876702067728256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763854.635296:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067728256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763854.635311:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763854.635317:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763854.635323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067728256:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763855.402060:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763856.426077:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763857.450003:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763858.474106:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763859.498009:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763859.753950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.753962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763859.753966:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.753979:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.753992:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754000:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763859.754042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067728384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763859.754075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754079:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763859.754081:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754089:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754093:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754103:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763859.754117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067728512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763859.754128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763859.754135:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754142:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754147:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754152:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763859.754160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067728384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763859.754169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067728640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763859.754186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763859.754191:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754194:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754199:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754202:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754207:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763859.754220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067728768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763859.754230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763859.754237:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754240:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754244:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754248:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754253:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763859.754281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763859.754281:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067728896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763859.754296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763859.754304:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754310:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754314:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754318:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754323:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763859.754338:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067729024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763859.754344:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa5f80 x1876702067728384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763859.754366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763859.754370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763859.754372:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763859.754379:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1327 00000100:00080000:3.0:1789763859.754385:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763859.754390:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763859.754396:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763859.754401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067728384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763859.754413:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067729152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763859.754422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754442:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067728384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763859.754452:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763859.754483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754502:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa6680 x1876702067728512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067728512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754525:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754529:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754533:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763859.754560:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763859.754579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754590:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa7100 x1876702067728640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754603:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067728640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754612:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754615:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763859.754625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763859.754643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa4700 x1876702067728768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754667:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067728768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754675:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754678:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754681:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763859.754687:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763859.754704:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754715:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa5500 x1876702067728896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067728896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067728896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763859.754746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763859.754764:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa4000 x1876702067729024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754848:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067729024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754864:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763859.754871:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763859.754896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763859.754907:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763859/real 1789763859] req@ffff936010aa5c00 x1876702067729152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763859.754919:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067729152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763859.754927:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763859.754930:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763859.754933:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729152:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763860.522053:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763861.546010:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763862.569910:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763863.594010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763864.617957:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763864.873947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.873962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763864.873968:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.873983:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.873996:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874005:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763864.874070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067729280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763864.874121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.874130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763864.874134:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874142:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874150:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874159:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763864.874177:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763864.874198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067729408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763864.874210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067729280:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789763864.874234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.874242:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763864.874245:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874252:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874259:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874265:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763864.874272:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763864.874290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067729536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763864.874304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.874308:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763864.874311:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874315:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874319:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874324:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763864.874333:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763864.874347:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067729664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763864.874361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.874366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763864.874368:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874375:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874380:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763864.874389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729408:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763864.874390:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763864.874402:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763864.874422:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067729792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763864.874436:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763864.874441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763864.874445:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874453:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874459:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874466:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763864.874473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729536:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789763864.874477:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763864.874501:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067729920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763864.874515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763864.874522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729664:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789763864.874524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763864.874532:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763864.874541:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1332 00000100:00080000:0.0:1789763864.874546:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763864.874552:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763864.874561:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763864.874578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763864.874585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067730048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763864.874624:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763864.874679:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730048:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763865.642031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763866.666004:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763867.690130:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763867.946169:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789763867.946246:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763867.946267:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946332:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946360:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946369:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946378:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946386:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789763867.946393:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789763867.946536:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020dae680 x1876702067730048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946577:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067730048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.946612:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.946622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.946629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763867.946681:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020daf480 x1876702067729792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067729792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.946754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.946761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.946766:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763867.946779:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020dac380 x1876702067729920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946831:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067729920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.946846:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.946852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.946857:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067729920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763867.946867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020daf100 x1876702067729408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067729408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.946901:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.946906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.946911:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763867.946922:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020dac700 x1876702067729536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946941:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067729536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.946955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.946960:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.946965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763867.946975:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020dad880 x1876702067729664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.946991:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067729664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.947003:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.947009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.947015:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067729664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763867.947025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763864/real 1789763867] req@ffff936020dad500 x1876702067729280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763867.947045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067729280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763867.947057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763867.947062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763867.947067:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067729280:192.168.203.137@tcp:250:kworker.0 00010000:00010000:1.0:1789763868.714047:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763869.738018:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763869.993956:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.993971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763869.993977:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763869.993993:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994019:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763869.994068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067730176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763869.994114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.994120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763869.994123:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763869.994129:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994142:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763869.994155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763869.994176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067730304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763869.994190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.994196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763869.994199:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763869.994206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994211:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994218:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763869.994229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067730176:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763869.994229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763869.994259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067730432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763869.994284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.994292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763869.994297:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763869.994306:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994331:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994342:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763869.994352:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763869.994354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763869.994389:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067730560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763869.994410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.994416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763869.994432:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763869.994439:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa7480 x1876702067730176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763869.994448:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994464:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994480:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763869.994492:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763869.994519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067730176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763869.994529:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067730688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763869.994567:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763869.994575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763869.994584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763869.994592:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763869.994601:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763869.994603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067730176:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763869.994621:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00080000:3.0:1789763869.994633:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763869.994636:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730304:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763869.994652:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763869.994666:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763869.994698:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067730816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763869.994743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.994785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa4e00 x1876702067730304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763869.994864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763869.994877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763869.994882:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763869.994896:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1337 00000100:00100000:2.0:1789763869.994901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067730304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763869.994905:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763869.994924:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763869.994943:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763869.994945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.994963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.994975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763869.994982:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067730944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763869.995040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763869.995096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.995125:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa6a00 x1876702067730432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763869.995152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067730432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763869.995182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.995189:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.995196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763869.995210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763869.995256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.995284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa5c00 x1876702067730560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763869.995311:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067730560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763869.995327:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.995334:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.995340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763869.995366:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763869.995410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.995436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa4000 x1876702067730688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763869.995462:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067730688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763869.995480:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.995487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.995494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067730688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763869.995506:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763869.995552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.995586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa5500 x1876702067730816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763869.995615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067730816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763869.995633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.995639:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.995646:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763869.995658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763869.995736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763869.995765:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763869/real 1789763869] req@ffff936010aa4700 x1876702067730944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763869.995790:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067730944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763869.995853:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763869.995862:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763869.995871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067730944:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789763870.762056:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763871.786019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763872.810037:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763873.833948:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763874.858079:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763875.113902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763875.113915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763875.113935:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.113944:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.113954:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.113959:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763875.113991:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067731072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763875.114012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763875.114015:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763875.114016:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114019:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114021:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114023:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114032:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763875.114041:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067731200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763875.114049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067731072:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763875.114052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763875.114057:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763875.114058:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114062:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114064:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114070:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763875.114077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067731328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763875.114084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763875.114087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763875.114088:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114091:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114101:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114112:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114115:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763875.114123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763875.114124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067731456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763875.114133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763875.114136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763875.114138:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114140:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114143:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114145:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114148:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763875.114156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067731584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763875.114162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763875.114165:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa7100 x1876702067731072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763875.114165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763875.114168:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114175:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114180:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114185:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114189:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763875.114194:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067731072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763875.114200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067731712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763875.114206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763875.114209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763875.114213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763875.114215:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763875.114219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067731072:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763875.114219:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763875.114229:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1342 00000100:00080000:3.0:1789763875.114234:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763875.114238:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763875.114244:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763875.114246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763875.114263:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067731840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763875.114270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa6680 x1876702067731200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114295:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067731200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763875.114309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763875.114320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa5f80 x1876702067731328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067731328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114344:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114346:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114348:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763875.114351:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763875.114361:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114368:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa5880 x1876702067731456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114374:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067731456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114380:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114382:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114383:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763875.114395:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763875.114405:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114411:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa4a80 x1876702067731584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114418:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067731584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114423:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114425:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114428:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067731584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763875.114431:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067731712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763875.114442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa4380 x1876702067731712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114467:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067731712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114474:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067731712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763875.114490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067731840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763875.114501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763875.114526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763875/real 1789763875] req@ffff936010aa5180 x1876702067731840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763875.114534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067731840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763875.114540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763875.114543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763875.114545:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067731840:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763875.882023:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763876.906053:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763877.930060:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763878.954043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763879.977963:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763880.233946:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.233959:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763880.233964:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.233978:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234004:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234014:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763880.234100:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067731968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234157:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234161:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763880.234163:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234167:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234170:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234173:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234184:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763880.234197:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067732096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763880.234211:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234217:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234220:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763880.234226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067731968:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763880.234227:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234238:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763880.234247:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067732224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234273:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234277:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763880.234279:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234295:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234299:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234303:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234309:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763880.234325:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067732352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234335:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234339:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763880.234342:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234345:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234348:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234352:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234356:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763880.234365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763880.234374:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067732480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234385:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234407:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763880.234410:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234415:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00080000:3.0:1789763880.234422:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234427:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234433:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763880.234449:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067732608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763880.234461:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763880.234465:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763880.234468:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763880.234477:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1348 00000100:00100000:2.0:1789763880.234479:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa6300 x1876702067731968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763880.234482:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763880.234495:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763880.234509:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763880.234530:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067732736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763880.234565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067731968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.234581:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.234587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.234592:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067731968:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763880.234618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763880.234690:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.234712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa7800 x1876702067732096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.234734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067732096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.234748:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.234754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.234759:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763880.234796:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763880.234861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.234875:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa7b80 x1876702067732224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.234888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067732224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.234908:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.234913:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.234918:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763880.234927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763880.234959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.234997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa5180 x1876702067732352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.235011:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067732352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.235019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.235022:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.235026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763880.235032:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763880.235054:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.235072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa4380 x1876702067732480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.235083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067732480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.235090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.235092:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.235112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763880.235117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067732608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763880.235132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.235140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa4a80 x1876702067732608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.235149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067732608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.235155:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.235157:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.235159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067732608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763880.235163:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067732736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763880.235175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763880.235182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763880/real 1789763880] req@ffff936010aa5880 x1876702067732736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763880.235191:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067732736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763880.235197:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763880.235199:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763880.235201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067732736:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763881.001986:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763882.026131:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763883.049990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763884.073956:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763885.098126:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763885.354043:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763885.354057:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763885.354062:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354076:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354089:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354099:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763885.354147:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067732864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763885.354191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763885.354196:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763885.354198:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354221:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354226:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354230:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354256:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763885.354275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067732992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763885.354294:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763885.354307:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763885.354311:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067732864:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763885.354313:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354337:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354342:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354347:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354353:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763885.354372:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067733120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763885.354385:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763885.354390:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763885.354394:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354415:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354426:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354432:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354439:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763885.354457:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067733248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763885.354471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763885.354482:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763885.354488:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763885.354493:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354503:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354509:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354515:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354522:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763885.354558:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067733376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763885.354574:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763885.354578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef6a00 x1876702067732864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763885.354580:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763885.354609:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354647:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354656:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354663:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354673:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763885.354684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067732864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763885.354715:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067733504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763885.354718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.354742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763885.354755:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763885.354764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067732864:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763885.354770:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763885.354783:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763885.354793:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1353 00000100:00080000:1.0:1789763885.354799:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763885.354805:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763885.354814:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763885.354888:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789763885.354910:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067733632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763885.354973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef5f80 x1876702067732992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067732992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355081:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355089:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355096:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067732992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763885.355112:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763885.355159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355191:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef6680 x1876702067733120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355211:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067733120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355231:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355236:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355240:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763885.355249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763885.355280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355294:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef7100 x1876702067733248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355309:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067733248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355319:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355323:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355326:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763885.355334:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763885.355356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef5500 x1876702067733376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355383:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067733376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355393:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355396:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763885.355409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067733504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763885.355441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef4e00 x1876702067733504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355467:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067733504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355480:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067733504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763885.355508:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067733632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763885.355531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763885.355553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763885/real 1789763885] req@ffff936010ef6300 x1876702067733632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763885.355568:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067733632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763885.355578:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763885.355581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763885.355585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067733632:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763886.122076:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763887.146039:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763888.170069:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763889.194072:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763890.218128:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763890.473933:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.473943:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763890.473946:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.473955:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.473965:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.473971:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763890.474005:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067733760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763890.474028:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474031:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763890.474032:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474035:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474037:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.474039:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474047:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763890.474055:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067733888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763890.474062:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474065:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763890.474067:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474069:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474071:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.474073:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474076:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763890.474083:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067734016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763890.474094:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474096:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763890.474098:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474100:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474103:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.474106:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474110:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763890.474118:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067734144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763890.474125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067733760:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763890.474129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763890.474142:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474146:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474149:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.474151:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474155:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763890.474165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067734272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763890.474171:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763890.474176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474178:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474182:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763890.474185:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474189:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763890.474198:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067734400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763890.474204:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763890.474206:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763890.474208:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763890.474212:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1358 00000100:00080000:3.0:1789763890.474215:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763890.474222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763890.474226:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763890.474231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763890.474241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067734528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763890.474285:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa5f80 x1876702067733760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474319:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067733760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474329:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067733760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763890.474342:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763890.474362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474371:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa6680 x1876702067733888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474380:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067733888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474386:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474388:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474390:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067733888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763890.474394:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763890.474407:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474414:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa7100 x1876702067734016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067734016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474433:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474435:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763890.474468:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763890.474480:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474488:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa4700 x1876702067734144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067734144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474502:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474504:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763890.474510:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763890.474521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474528:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa5500 x1876702067734272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067734272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763890.474568:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067734400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763890.474582:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474606:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa4000 x1876702067734400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067734400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474624:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067734400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763890.474627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067734528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763890.474640:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763890.474646:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763890/real 1789763890] req@ffff936010aa5c00 x1876702067734528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763890.474653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067734528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763890.474658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763890.474660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763890.474662:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067734528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763891.242108:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763892.266165:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763893.290068:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763894.314064:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763895.338055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763895.593926:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.593940:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763895.593945:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.593959:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.593973:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.593982:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763895.594029:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067734656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763895.594076:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594081:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763895.594083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594088:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594093:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594109:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763895.594123:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067734784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763895.594136:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763895.594143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594148:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594151:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594155:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594161:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763895.594171:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067734912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763895.594182:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763895.594190:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594195:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594199:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594204:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763895.594224:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067735040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763895.594239:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067734656:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763895.594247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763895.594259:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594265:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594269:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594275:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594283:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763895.594297:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067735168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763895.594308:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594312:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763895.594315:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594320:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594338:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594343:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594351:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763895.594356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763895.594374:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067735296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763895.594397:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763895.594402:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763895.594405:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763895.594411:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1363 00000100:00080000:3.0:1789763895.594418:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763895.594423:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763895.594430:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763895.594432:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa6a00 x1876702067734656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763895.594451:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067735424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763895.594469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067734656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594483:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594488:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067734656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763895.594501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763895.594555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa4e00 x1876702067734784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067734784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594587:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594590:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763895.594606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763895.594623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594632:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa7480 x1876702067734912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067734912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067734912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763895.594700:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763895.594716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa5c00 x1876702067735040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067735040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763895.594752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763895.594766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594775:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa4000 x1876702067735168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067735168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594790:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594848:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763895.594855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067735296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763895.594872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594882:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa5500 x1876702067735296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594893:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067735296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067735296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763895.594909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067735424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763895.594923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763895.594931:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763895/real 1789763895] req@ffff936010aa4700 x1876702067735424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763895.594940:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067735424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763895.594946:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763895.594949:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763895.594951:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067735424:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763896.362094:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763897.386134:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763898.410108:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763899.434109:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763900.458133:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763900.713970:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.713984:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763900.713990:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714004:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714018:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714028:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763900.714091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067735552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763900.714129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.714134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763900.714137:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714141:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714146:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714150:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714161:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763900.714176:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067735680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763900.714188:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.714192:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763900.714195:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714199:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714209:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714215:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763900.714226:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067735808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763900.714238:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.714243:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763900.714247:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714252:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714256:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714261:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714268:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763900.714284:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067735936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763900.714295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067735552:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763900.714310:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.714316:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763900.714318:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714326:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714329:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714336:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714342:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763900.714358:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067736064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763900.714370:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763900.714375:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763900.714378:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714383:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714389:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714395:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714400:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763900.714414:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067736192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763900.714426:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763900.714437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763900.714440:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763900.714445:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763900.714453:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1368 00000100:00080000:1.0:1789763900.714460:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763900.714465:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763900.714472:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763900.714527:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067736320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763900.714554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef6d80 x1876702067735552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.714592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067735552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.714609:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.714616:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.714622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067735552:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763900.714633:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763900.714665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.714682:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef7480 x1876702067735680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.714698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067735680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.714708:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.714712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.714717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763900.714724:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763900.714746:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.714760:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef7b80 x1876702067735808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.714774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067735808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.714794:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.714845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.714849:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763900.714889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763900.714918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.714933:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef7800 x1876702067735936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.714948:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067735936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.714959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.714963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.714967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067735936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763900.714974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763900.714995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.715008:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef4380 x1876702067736064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.715022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067736064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.715031:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.715035:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.715040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763900.715047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067736192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763900.715069:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.715082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef4000 x1876702067736192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.715095:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067736192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.715105:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.715108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.715112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067736192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763900.715118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067736320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763900.715139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763900.715151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763900/real 1789763900] req@ffff936010ef5c00 x1876702067736320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763900.715164:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067736320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763900.715174:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763900.715177:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763900.715181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067736320:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763901.482086:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763902.506062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763903.530072:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763904.554120:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763905.578131:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763905.834001:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834016:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763905.834021:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834035:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834049:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834058:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763905.834107:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067736448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763905.834153:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834158:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763905.834162:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834167:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834172:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834176:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834188:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763905.834204:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067736576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763905.834219:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834224:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763905.834227:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834231:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834235:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834239:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834245:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763905.834262:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067736448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763905.834263:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067736704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763905.834291:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834295:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763905.834297:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834302:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834306:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834311:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834317:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763905.834335:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067736832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763905.834347:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834351:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763905.834354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834358:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834363:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834367:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763905.834383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763905.834398:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067736960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763905.834413:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763905.834418:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763905.834420:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763905.834424:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834429:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834460:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834471:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763905.834477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef4700 x1876702067736448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763905.834505:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067737088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763905.834514:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067736448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.834560:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763905.834566:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763905.834570:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.834582:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067736448:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763905.834584:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763905.834596:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763905.834600:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736576:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763905.834613:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1373 00000100:00080000:1.0:1789763905.834624:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763905.834634:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763905.834646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:1.0:1789763905.834647:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763905.834686:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067737216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763905.834687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef5180 x1876702067736576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.834761:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067736576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.834776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.834780:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.834786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763905.834888:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763905.834923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763905.834945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef4a80 x1876702067736704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.834963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067736704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.834984:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.834990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.834995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763905.835003:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763905.835033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763905.835079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef5880 x1876702067736832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.835115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067736832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.835135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.835143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.835151:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763905.835165:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763905.835205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763905.835237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef5c00 x1876702067736960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.835273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067736960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.835294:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.835302:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.835311:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067736960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763905.835327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763905.835369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763905.835388:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef4000 x1876702067737088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.835404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067737088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.835416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.835421:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.835426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763905.835433:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763905.835497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763905.835513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763905/real 1789763905] req@ffff936010ef4380 x1876702067737216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763905.835529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067737216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763905.835540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763905.835544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763905.835548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737216:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763906.602021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763907.625914:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763908.650147:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763909.674045:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763910.698061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763910.954018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763910.954031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763910.954038:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954053:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954072:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954083:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763910.954144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067737344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763910.954193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763910.954196:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954202:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954207:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954212:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954229:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763910.954246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067737472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763910.954265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763910.954267:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954272:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954276:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954280:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954286:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763910.954297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067737344:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763910.954311:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067737600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954331:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763910.954338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763910.954341:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954364:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954370:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954375:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954389:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763910.954430:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067737728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763910.954451:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763910.954460:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763910.954465:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954474:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954483:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954488:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954497:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763910.954514:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067737856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763910.954534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa7100 x1876702067737344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763910.954535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763910.954541:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763910.954563:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954578:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763910.954593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954600:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763910.954611:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067737344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763910.954627:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067737984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763910.954641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763910.954647:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763910.954649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763910.954651:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763910.954694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763910.954694:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1378 00000100:00080000:3.0:1789763910.954715:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763910.954719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067737344:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763910.954731:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763910.954741:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763910.954748:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763910.954765:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067738112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763910.954847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.954874:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa6680 x1876702067737472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.954903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067737472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.954923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.954931:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.954945:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763910.954991:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763910.955025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.955041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa5f80 x1876702067737600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.955081:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067737600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.955108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.955115:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.955122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763910.955136:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763910.955177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.955200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa5880 x1876702067737728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.955225:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067737728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.955242:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.955248:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.955255:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763910.955265:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763910.955295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.955310:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa4a80 x1876702067737856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.955325:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067737856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.955335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.955339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.955343:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067737856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763910.955350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763910.955371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.955382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa4380 x1876702067737984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.955395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067737984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.955430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.955434:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.955438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067737984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763910.955446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067738112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763910.955473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763910.955487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763910/real 1789763910] req@ffff936010aa5180 x1876702067738112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763910.955506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067738112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763910.955521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763910.955527:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763910.955532:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067738112:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763911.722140:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763912.746098:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763913.770050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763914.794113:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763915.818050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763916.073910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.073931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763916.073937:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.073951:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.073965:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.073974:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763916.074020:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067738240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074062:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763916.074065:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074070:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.074074:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.074079:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074091:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763916.074106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067738368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074125:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763916.074133:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074143:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00100000:2.0:1789763916.074145:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067738240:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763916.074147:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.074159:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074172:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763916.074186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067738496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074198:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763916.074205:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074209:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.074213:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.074218:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074224:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763916.074238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067738624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763916.074256:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074260:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.074266:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763916.074276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763916.074277:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074290:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763916.074305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067738752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074316:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074320:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763916.074322:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074327:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.074332:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.074336:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074341:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763916.074357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067738880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763916.074393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763916.074402:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763916.074405:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa7b80 x1876702067738240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763916.074405:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763916.074417:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1383 00000100:00080000:3.0:1789763916.074427:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763916.074438:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763916.074451:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763916.074479:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067738240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763916.074479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067739008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763916.074547:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.074556:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.074563:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067738240:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763916.074601:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763916.074641:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.074682:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa7800 x1876702067738368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.074711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067738368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.074729:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.074737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.074744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763916.074757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763916.074794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.074853:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa6300 x1876702067738496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.074870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067738496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.074890:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.074894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.074899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763916.074907:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763916.074933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.074947:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa5180 x1876702067738624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.074961:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067738624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.074986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.074990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.074995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763916.075002:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763916.075024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.075038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa4380 x1876702067738752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.075052:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067738752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.075062:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.075066:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.075071:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067738752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763916.075078:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067738880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763916.075099:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.075112:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa4a80 x1876702067738880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.075126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067738880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.075136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.075140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.075144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067738880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763916.075151:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763916.075171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763916.075183:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763916/real 1789763916] req@ffff936010aa5880 x1876702067739008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763916.075198:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067739008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763916.075207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763916.075211:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763916.075214:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763916.842113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763917.866022:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763918.889945:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763919.914079:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763920.938035:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763921.194005:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194021:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763921.194029:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763921.194047:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194064:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763921.194075:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763921.194146:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067739136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763921.194201:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194207:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763921.194210:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763921.194215:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194220:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763921.194225:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763921.194239:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763921.194254:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067739264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763921.194279:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194285:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763921.194287:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763921.194293:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194297:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763921.194301:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763921.194305:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763921.194321:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067739392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763921.194365:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194383:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763921.194388:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763921.194394:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067739136:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763921.194396:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194409:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763921.194425:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763921.194433:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763921.194457:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067739520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763921.194479:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194486:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763921.194489:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763921.194498:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194507:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763921.194516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763921.194523:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763921.194541:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763921.194571:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067739648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763921.194585:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763921.194591:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763921.194593:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763921.194605:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00100000:2.0:1789763921.194621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef7800 x1876702067739136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763921.194623:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763921.194632:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763921.194643:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763921.194676:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067739776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763921.194681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067739136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763921.194729:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763921.194736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763921.194740:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763921.194745:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763921.194748:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763921.194757:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1389 00000100:00080000:1.0:1789763921.194767:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763921.194768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067739136:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763921.194790:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763921.194816:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739264:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789763921.194857:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763921.194891:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067739904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763921.194951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.194976:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef7b80 x1876702067739264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.194994:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067739264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195005:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763921.195038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763921.195059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.195068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef7480 x1876702067739392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.195078:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067739392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195092:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195095:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763921.195100:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763921.195113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.195121:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef6d80 x1876702067739520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.195129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067739520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195137:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763921.195144:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763921.195156:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.195164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef6300 x1876702067739648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.195172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067739648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195177:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195179:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195182:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067739648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763921.195186:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763921.195199:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.195206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef4e00 x1876702067739776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.195213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067739776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195224:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763921.195227:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763921.195239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763921.195246:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763921/real 1789763921] req@ffff936010ef5500 x1876702067739904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763921.195253:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067739904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763921.195259:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763921.195261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763921.195264:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067739904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763921.962056:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763922.986135:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763924.010012:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763925.034071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763926.058092:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763926.313932:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763926.313945:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763926.313951:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.313966:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.313994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314005:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763926.314075:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067740032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314118:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763926.314123:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763926.314126:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314141:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314146:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314150:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314165:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763926.314180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067740160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314194:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763926.314199:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763926.314202:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314208:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314212:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314217:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314224:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763926.314237:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067740288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763926.314256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067740032:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763926.314261:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763926.314269:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314277:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314280:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314307:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314315:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763926.314336:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067740416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314350:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763926.314355:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763926.314358:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314364:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314381:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314387:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314392:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763926.314403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763926.314411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067740544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314427:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763926.314432:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763926.314436:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314441:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314446:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314451:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314456:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763926.314473:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067740672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763926.314486:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763926.314488:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa5f80 x1876702067740032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763926.314493:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763926.314499:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763926.314514:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1394 00000100:00080000:3.0:1789763926.314524:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763926.314530:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763926.314537:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763926.314542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067740032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763926.314556:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067740800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763926.314565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.314580:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.314590:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067740032:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763926.314603:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763926.314638:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.314655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa6680 x1876702067740160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.314670:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067740160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.314681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.314685:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.314689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763926.314725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763926.314747:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.314761:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa7100 x1876702067740288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.314776:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067740288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.314869:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.314877:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.314885:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763926.314898:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763926.314939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.314957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa4700 x1876702067740416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.314973:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067740416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.314989:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.314993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.314997:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763926.315003:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763926.315026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.315039:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa5500 x1876702067740544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.315053:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067740544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.315063:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.315066:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.315071:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067740544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763926.315085:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067740672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763926.315106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.315119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa4000 x1876702067740672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.315132:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067740672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.315140:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.315144:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.315148:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067740672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763926.315155:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067740800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763926.315176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763926.315188:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763926/real 1789763926] req@ffff936010aa5c00 x1876702067740800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763926.315202:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067740800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763926.315211:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763926.315215:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763926.315218:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067740800:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763927.082044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763928.106047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763929.130002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763930.154052:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763931.178030:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763931.433941:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.433960:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763931.433966:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.433980:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.433994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434005:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763931.434053:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067740928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434097:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763931.434105:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434110:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.434115:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434120:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434132:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763931.434147:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067741056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434161:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434166:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763931.434169:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434174:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.434178:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434182:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434188:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763931.434202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067740928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763931.434204:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067741184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434235:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763931.434237:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434242:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.434247:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434283:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434304:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763931.434333:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067741312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434355:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434362:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763931.434366:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434374:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.434383:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434392:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434402:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763931.434403:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763931.434429:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067741440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434446:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434450:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763931.434453:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434459:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00080000:3.0:1789763931.434466:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434471:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434480:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763931.434500:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763931.434501:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067741568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763931.434540:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763931.434545:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763931.434549:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763931.434559:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1399 00000100:00100000:2.0:1789763931.434560:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741312:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763931.434565:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763931.434588:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763931.434596:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763931.434615:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067741696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763931.434633:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763931.434691:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067741568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763931.434736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067741696:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763932.202015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763933.226044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763934.250107:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789763934.506267:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789763934.506347:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789763934.506369:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763934.506412:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763934.506430:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789763934.506439:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa7480 x1876702067740928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763934.506445:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763934.506462:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789763934.506477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067740928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789763934.506477:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789763934.506492:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:0.0:1789763934.506496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506502:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506509:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067740928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763934.506524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa4e00 x1876702067741056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067741056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506574:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.506616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa5500 x1876702067741568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506635:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067741568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506647:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506658:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067741568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.506668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa4700 x1876702067741696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067741696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506697:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506706:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067741696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.506716:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa6a00 x1876702067741184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067741184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506753:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763934.506774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa5c00 x1876702067741312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506793:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067741312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506849:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763934.506866:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763931/real 1789763934] req@ffff936010aa4000 x1876702067741440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763934.506888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067741440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763934.506899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763934.506906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763934.506914:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741440:192.168.203.137@tcp:8:kworker.0 00010000:00010000:3.0:1789763935.274094:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763936.298081:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763936.554037:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554051:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763936.554056:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554070:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554084:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554092:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763936.554135:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067741824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763936.554172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554177:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763936.554181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554186:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554190:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554195:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763936.554208:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763936.554259:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067741952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763936.554267:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067741824:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763936.554293:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554299:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763936.554303:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554313:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554322:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554331:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763936.554354:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763936.554381:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067742080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763936.554411:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763936.554417:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554426:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763936.554430:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554452:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554461:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554470:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763936.554481:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763936.554489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef7100 x1876702067741824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763936.554515:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067742208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763936.554538:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067741824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763936.554539:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554560:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763936.554570:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554589:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554605:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763936.554615:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763936.554620:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789763936.554632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789763936.554639:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763936.554647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067741824:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789763936.554675:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067742336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763936.554701:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763936.554702:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741952:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763936.554723:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763936.554731:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554741:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554750:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554759:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763936.554771:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763936.554787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763936.554845:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067742464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763936.554868:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763936.554876:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763936.554880:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763936.554888:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1404 00000100:00080000:1.0:1789763936.554897:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763936.554906:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763936.554917:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763936.554924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef6680 x1876702067741952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763936.554946:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067742592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763936.554966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067741952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.554988:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.554997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067741952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763936.555023:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763936.555070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763936.555094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef5f80 x1876702067742080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763936.555120:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067742080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.555138:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.555145:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555151:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763936.555165:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763936.555208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763936.555261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef6a00 x1876702067742208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763936.555284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067742208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.555300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.555306:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555312:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763936.555345:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763936.555388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763936.555414:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef5500 x1876702067742336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763936.555442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067742336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.555460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.555467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555473:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763936.555486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067742464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763936.555527:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763936.555552:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef4e00 x1876702067742464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763936.555577:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067742464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.555594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.555600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555606:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067742464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763936.555619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067742592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763936.555659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763936.555682:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763936/real 1789763936] req@ffff936010ef6300 x1876702067742592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763936.555707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067742592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763936.555721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763936.555727:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763936.555733:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067742592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763937.322079:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763938.346002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763939.370073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763940.394143:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763941.418038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763941.674003:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674016:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763941.674022:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674039:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674069:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763941.674080:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763941.674125:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067742720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763941.674163:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674169:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763941.674173:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674200:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674207:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763941.674215:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763941.674221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067742720:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789763941.674232:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763941.674261:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067742848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763941.674280:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674286:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763941.674290:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674297:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674306:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763941.674313:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763941.674322:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763941.674334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763941.674346:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067742976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763941.674363:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674385:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763941.674388:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674396:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674404:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763941.674408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef6d80 x1876702067742720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763941.674416:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763941.674433:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763941.674442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067742720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.674457:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789763941.674458:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067743104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763941.674465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.674475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067742720:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763941.674493:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674499:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763941.674502:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674508:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674515:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763941.674516:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763941.674526:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763941.674534:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763941.674552:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067743232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763941.674564:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763941.674568:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763941.674572:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763941.674575:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674579:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00080000:1.0:1789763941.674584:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763941.674585:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef7480 x1876702067742848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763941.674592:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763941.674604:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763941.674614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067742848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763941.674622:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067743360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763941.674632:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763941.674645:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763941.674646:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789763941.674655:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763941.674656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763941.674659:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763941.674671:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1409 00000100:00100000:2.0:1789763941.674673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742976:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789763941.674676:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763941.674686:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763941.674711:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763941.674727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763941.674733:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067743488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763941.674747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef7b80 x1876702067742976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763941.674766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067742976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.674777:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763941.674782:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.674788:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067742976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763941.674797:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763941.674822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763941.674876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef7800 x1876702067743104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763941.674890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067743104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.674899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763941.674902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.674907:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763941.674929:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763941.674958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763941.674975:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef4380 x1876702067743232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763941.674992:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067743232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.675002:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763941.675004:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.675008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763941.675013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067743360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763941.675034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763941.675044:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef4000 x1876702067743360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763941.675054:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067743360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.675068:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763941.675070:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.675073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067743360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763941.675077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067743488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763941.675091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763941.675099:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763941/real 1789763941] req@ffff936010ef5c00 x1876702067743488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763941.675108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067743488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763941.675115:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763941.675117:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763941.675120:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067743488:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763942.442067:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763943.466207:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763944.490178:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763945.514031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763946.538064:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763946.793907:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.793919:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763946.793940:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.793952:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.793962:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.793970:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763946.794042:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067743616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794077:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794082:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763946.794085:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.794095:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794099:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794104:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763946.794115:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763946.794129:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067743744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794163:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794167:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763946.794169:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.794174:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794187:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794212:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763946.794217:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067743616:192.168.203.137@tcp:250:kworker.0 00000008:00100000:2.0:1789763946.794219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763946.794236:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067743872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794251:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763946.794253:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.794256:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794259:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794263:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763946.794268:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763946.794279:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067744000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794288:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794291:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763946.794294:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.794297:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794300:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794303:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763946.794307:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763946.794314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763946.794321:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067744128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794331:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794335:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763946.794338:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763946.794342:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794345:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794348:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763946.794352:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763946.794362:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067744256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763946.794371:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763946.794374:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763946.794380:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763946.794382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061dd500 x1876702067743616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763946.794386:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1414 00000100:00080000:2.0:1789763946.794409:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763946.794417:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763946.794427:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763946.794445:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067744384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763946.794458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067743616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.794496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.794503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.794511:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067743616:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763946.794523:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763946.794559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.794578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061df480 x1876702067743744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.794613:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067743744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.794628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.794633:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.794639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763946.794674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763946.794709:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.794729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061df100 x1876702067743872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.794751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067743872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.794766:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.794850:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.794856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067743872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763946.794919:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763946.794989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.795019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061dd880 x1876702067744000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.795043:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067744000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.795057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.795063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.795068:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763946.795078:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763946.795111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.795130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061ddc00 x1876702067744128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.795171:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067744128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.795194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.795200:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.795205:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763946.795215:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067744256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763946.795248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.795268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061de300 x1876702067744256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.795288:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067744256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.795302:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.795307:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.795312:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067744256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763946.795329:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067744384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763946.795362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763946.795380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763946/real 1789763946] req@ffff9360061ddf80 x1876702067744384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763946.795401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067744384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763946.795413:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763946.795418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763946.795423:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067744384:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763947.562054:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763948.586110:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763949.609986:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763950.634047:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763951.658058:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763951.913989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763951.914005:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914016:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914027:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763951.914048:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763951.914093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067744512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763951.914153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763951.914159:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914163:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914168:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763951.914173:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763951.914183:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763951.914195:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067744512:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763951.914200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067744640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763951.914213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763951.914218:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914222:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914225:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763951.914227:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763951.914233:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763951.914244:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067744768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763951.914253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763951.914258:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914260:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914265:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763951.914270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789763951.914272:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789763951.914279:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763951.914291:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067744896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763951.914299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789763951.914304:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914307:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914311:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763951.914317:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020dafb80 x1876702067744512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763951.914322:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763951.914338:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067745024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763951.914347:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067744512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763951.914349:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763951.914387:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914399:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:2.0:1789763951.914400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763951.914407:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763951.914413:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789763951.914421:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067744512:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789763951.914431:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763951.914450:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067745152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763951.914466:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744640:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789763951.914483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763951.914489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763951.914492:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763951.914500:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1419 00000100:00080000:0.0:1789763951.914506:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763951.914511:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789763951.914519:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789763951.914520:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763951.914543:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067745280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763951.914546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020dac000 x1876702067744640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.914598:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067744640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.914629:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.914634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.914639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763951.914646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763951.914668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020daed80 x1876702067744768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.914695:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067744768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.914705:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.914708:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.914712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763951.914718:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763951.914737:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914749:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020dad500 x1876702067744896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.914763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067744896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.914772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.914775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.914780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067744896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763951.914786:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763951.914846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914861:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020dad880 x1876702067745024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.914875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067745024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.914885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.914888:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.914892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763951.914912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067745152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763951.914935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763951.914948:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020dac700 x1876702067745152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.914962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067745152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.914971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.914975:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.914978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067745152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763951.914985:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067745280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763951.915004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763951.915016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763951/real 1789763951] req@ffff936020daf100 x1876702067745280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763951.915030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067745280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763951.915039:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763951.915043:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763951.915049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067745280:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763952.682133:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763953.705947:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763954.729971:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763955.754105:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763956.778056:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763957.034014:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034027:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763957.034033:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034046:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034060:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034098:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763957.034148:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067745408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763957.034185:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034198:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763957.034201:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034206:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034210:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034215:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034226:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763957.034240:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067745536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763957.034253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034258:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763957.034260:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034265:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034270:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034275:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034281:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763957.034303:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067745664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763957.034318:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034322:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763957.034325:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763957.034333:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067745408:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763957.034335:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034344:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034353:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034359:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763957.034387:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067745792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763957.034399:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034404:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763957.034406:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034411:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034417:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034420:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034428:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763957.034445:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067745920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763957.034452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789763957.034471:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034476:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763957.034479:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034486:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034490:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763957.034495:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034501:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763957.034526:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067746048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763957.034535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef5880 x1876702067745408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763957.034552:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763957.034568:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763957.034571:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763957.034579:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1424 00000100:00080000:1.0:1789763957.034586:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763957.034587:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067745408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763957.034597:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763957.034610:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763957.034621:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789763957.034630:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067746176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763957.034634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.034649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067745408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763957.034668:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763957.034699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.034716:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef4a80 x1876702067745536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.034731:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067745536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.034751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.034755:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.034760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763957.034793:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763957.034866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.034888:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef5180 x1876702067745664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.034903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067745664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.034914:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.034918:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.034922:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763957.034929:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763957.034961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.034973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef4700 x1876702067745792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.034988:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067745792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.034998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.035001:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.035006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763957.035013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789763957.035032:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.035044:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef5c00 x1876702067745920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.035058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067745920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.035104:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.035108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.035112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067745920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763957.035120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763957.035145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.035159:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef4000 x1876702067746048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.035174:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067746048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.035183:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.035187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.035191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763957.035197:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763957.035217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763957.035230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763957/real 1789763957] req@ffff936010ef4380 x1876702067746176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763957.035244:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067746176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763982 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763957.035254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763957.035257:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763957.035261:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763957.802013:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763958.826029:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763959.850164:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763960.874100:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763961.898108:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763962.153923:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.153932:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763962.153937:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763962.153948:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.153961:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.153970:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789763962.154020:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067746304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154097:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154103:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763962.154106:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763962.154112:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154128:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154136:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154154:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763962.154173:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067746432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154213:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763962.154217:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763962.154220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067746304:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789763962.154226:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154235:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154243:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154249:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763962.154263:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067746560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154275:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154279:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763962.154282:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763962.154287:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154290:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154313:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154320:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763962.154334:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067746688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154345:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154350:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789763962.154352:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763962.154357:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154363:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154367:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154373:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763962.154383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789763962.154394:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067746816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154422:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154424:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763962.154426:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763962.154429:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154433:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154436:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154439:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763962.154448:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067746944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763962.154454:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763962.154456:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763962.154459:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763962.154460:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef7800 x1876702067746304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763962.154464:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1429 00000100:00080000:1.0:1789763962.154477:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763962.154480:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789763962.154484:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763962.154485:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067746304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763962.154495:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067747072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763962.154498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154508:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154513:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067746304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763962.154538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763962.154558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef7b80 x1876702067746432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154588:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067746432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763962.154602:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763962.154623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154630:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef7480 x1876702067746560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067746560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154644:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154646:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763962.154652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763962.154666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef6d80 x1876702067746688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154688:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067746688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763962.154706:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763962.154717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154723:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef6300 x1876702067746816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067746816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154736:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154739:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067746816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763962.154742:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763962.154753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef4e00 x1876702067746944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067746944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154773:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067746944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763962.154789:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763962.154822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763962.154830:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763962/real 1789763962] req@ffff936010ef5500 x1876702067747072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763962.154837:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067747072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763962.154842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763962.154844:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763962.154846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747072:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789763962.922150:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763963.946147:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763964.970082:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763965.994157:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763967.018071:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763967.273975:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274014:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763967.274023:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274043:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274067:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274079:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763967.274147:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067747200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274193:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274199:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763967.274202:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274207:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274212:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274217:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274232:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763967.274248:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067747328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274264:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274270:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763967.274273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274278:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274282:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274287:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274292:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763967.274310:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067747456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274325:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274330:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763967.274333:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274343:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274347:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763967.274350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067747200:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763967.274354:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274368:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763967.274389:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067747584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274403:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274407:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763967.274410:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274416:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274422:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274427:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274433:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763967.274447:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067747712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274457:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274462:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763967.274464:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274470:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274475:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274481:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274486:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763967.274499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763967.274520:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067747840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763967.274536:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763967.274541:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763967.274544:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763967.274550:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1435 00000100:00080000:2.0:1789763967.274556:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763967.274561:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763967.274567:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763967.274589:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067747968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763967.274593:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061dc700 x1876702067747200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.274688:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067747200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.274713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.274720:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.274726:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067747200:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763967.274740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763967.274785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.274863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061dc380 x1876702067747328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.274882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067747328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.274893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.274898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.274902:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763967.274936:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763967.274963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.274977:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061ddf80 x1876702067747456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.275023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067747456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.275035:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.275040:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.275044:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763967.275053:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763967.275081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.275096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061de300 x1876702067747584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.275119:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067747584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.275129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.275133:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.275136:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763967.275143:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763967.275164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.275177:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061ddc00 x1876702067747712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.275191:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067747712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.275200:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.275204:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.275207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067747712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763967.275214:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763967.275238:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.275257:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061dd880 x1876702067747840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.275279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067747840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.275307:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.275313:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.275320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763967.275331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763967.275369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763967.275391:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763967/real 1789763967] req@ffff9360061df100 x1876702067747968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763967.275414:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067747968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763992 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763967.275431:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763967.275438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763967.275445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067747968:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763968.042126:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763969.065942:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763970.090076:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763971.114066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763972.137975:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763972.393907:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.393928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763972.393931:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.393941:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.393963:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.393972:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763972.394012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067748096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763972.394042:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.394045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763972.394047:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394054:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394057:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394066:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763972.394077:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067748224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763972.394087:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067748096:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763972.394088:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.394097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763972.394098:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394105:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394107:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394112:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763972.394120:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067748352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763972.394128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.394131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763972.394132:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394134:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394136:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394139:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394142:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763972.394149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067748480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763972.394154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763972.394162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.394165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763972.394167:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394171:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394174:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394176:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394179:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763972.394187:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067748608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763972.394193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763972.394197:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061df480 x1876702067748096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763972.394197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763972.394201:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394209:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394213:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394216:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394219:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763972.394220:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067748096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763972.394229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067748736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763972.394231:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763972.394241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763972.394246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763972.394247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067748096:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763972.394250:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763972.394256:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1440 00000100:00080000:2.0:1789763972.394257:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763972.394260:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763972.394263:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763972.394271:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067748864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763972.394276:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763972.394306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394322:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061dd500 x1876702067748224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067748224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394340:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763972.394349:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763972.394364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394371:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061dca80 x1876702067748352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067748352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394384:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394388:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763972.394391:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763972.394402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061df800 x1876702067748480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067748480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394425:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763972.394428:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763972.394438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061dfb80 x1876702067748608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394452:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067748608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394458:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067748608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763972.394463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067748736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763972.394474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061de680 x1876702067748736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067748736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394493:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394495:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067748736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763972.394507:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067748864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763972.394518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763972.394524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763972/real 1789763972] req@ffff9360061dc000 x1876702067748864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763972.394531:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067748864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763997 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763972.394536:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763972.394538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763972.394540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067748864:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763973.161918:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763974.185956:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763975.209973:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763976.233945:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763977.258038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763977.513963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763977.513976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763977.513983:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.513997:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00080000:3.0:1789763977.514013:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.514025:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763977.514080:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067748992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763977.514124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763977.514132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763977.514135:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.514145:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00080000:3.0:1789763977.514152:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.514160:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763977.514179:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763977.514194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067748992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789763977.514206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067749120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763977.514237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763977.514244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763977.514248:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.514256:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00080000:3.0:1789763977.514287:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.514307:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763977.514317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763977.514325:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763977.514343:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067749248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763977.514360:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763977.514367:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763977.514370:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.514376:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00080000:3.0:1789763977.514385:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.514392:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763977.514402:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa7100 x1876702067748992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763977.514455:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067748992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.514473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.514510:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.514517:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067748992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763977.514558:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763977.514616:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763977.514655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa6680 x1876702067749120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763977.514678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067749120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.514695:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.514701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.514708:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763977.514721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749248:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789763977.514736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763977.514761:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789763977.514762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067749376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763977.514782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763977.514789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa5f80 x1876702067749248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763977.514789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789763977.514876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067749248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.514902:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.514934:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.514944:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749248:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763977.514944:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.514953:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00080000:3.0:1789763977.514964:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.514972:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763977.514983:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763977.514984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789763977.515005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067749504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763977.515024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763977.515025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763977.515031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763977.515034:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.515040:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00100000:2.0:1789763977.515045:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa5880 x1876702067749376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763977.515047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.515054:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763977.515062:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763977.515069:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067749376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763977.515079:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067749632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763977.515085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.515091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789763977.515097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763977.515098:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749376:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763977.515103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763977.515106:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763977.515112:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1445 00000100:00100000:2.0:1789763977.515117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749504:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789763977.515118:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763977.515125:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763977.515132:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763977.515148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067749760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763977.515150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763977.515169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa4a80 x1876702067749504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763977.515190:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067749504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.515206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.515211:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.515217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067749504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789763977.515229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067749632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763977.515263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763977.515299:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa4380 x1876702067749632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763977.515324:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067749632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.515335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.515338:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.515342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067749632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789763977.515360:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067749760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789763977.515383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789763977.515395:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763977/real 1789763977] req@ffff936010aa5180 x1876702067749760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763977.515409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067749760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763977.515419:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763977.515422:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789763977.515426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067749760:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763978.282021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763979.305971:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763980.329966:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763981.354088:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763982.378108:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763982.634014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763982.634036:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634051:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634069:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634081:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763982.634155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067749888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763982.634210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634215:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763982.634218:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634223:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634228:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634232:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634248:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763982.634271:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067750016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763982.634292:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763982.634304:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634314:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634322:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634333:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634344:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763982.634362:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067749888:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789763982.634373:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067750144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763982.634391:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634396:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763982.634400:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634406:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634412:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634418:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634436:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763982.634457:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067750272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763982.634477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:0.0:1789763982.634491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763982.634494:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634510:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634521:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634531:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634541:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763982.634567:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067750400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763982.634580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763982.634586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061dd180 x1876702067749888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763982.634587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763982.634594:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634609:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00080000:2.0:1789763982.634620:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634629:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763982.634642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067749888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763982.634657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067750528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763982.634670:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.634690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789763982.634695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763982.634703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763982.634705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067749888:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763982.634707:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763982.634725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1450 00000100:00100000:0.0:1789763982.634732:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750016:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789763982.634733:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763982.634747:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763982.634755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763982.634776:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067750656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763982.634791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.634901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061dce00 x1876702067750016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.634921:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067750016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.634933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.634938:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.634942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763982.634980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763982.635009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.635024:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061dc000 x1876702067750144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.635039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067750144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.635050:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.635054:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.635058:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763982.635064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763982.635086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.635099:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061de680 x1876702067750272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.635124:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067750272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.635134:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.635137:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.635141:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763982.635147:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763982.635167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.635179:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061dfb80 x1876702067750400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.635193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067750400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.635203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.635206:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.635209:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763982.635215:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067750528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763982.635236:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.635249:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061df800 x1876702067750528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.635262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067750528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.635272:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.635276:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.635279:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067750528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763982.635285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067750656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763982.635306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763982.635318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763982/real 1789763982] req@ffff9360061dca80 x1876702067750656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763982.635331:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067750656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763982.635340:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763982.635343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763982.635347:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067750656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763983.402055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763984.426016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763985.449978:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763986.473972:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763987.497993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763987.753940:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.753951:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763987.753957:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.753970:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.753984:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.753992:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789763987.754037:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067750784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754065:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763987.754069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754072:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754075:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.754078:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754087:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763987.754096:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067750912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754104:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754107:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763987.754108:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754111:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754113:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.754115:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754118:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763987.754124:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067751040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754131:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754135:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789763987.754136:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067750784:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789763987.754137:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754142:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754146:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.754149:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754153:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763987.754160:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067751168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754166:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754169:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789763987.754171:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754173:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754175:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.754178:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754181:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763987.754187:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067751296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754192:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754195:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763987.754196:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754199:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754201:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763987.754204:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754206:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763987.754211:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067751424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763987.754217:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763987.754219:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763987.754220:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763987.754222:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1455 00000100:00080000:2.0:1789763987.754225:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763987.754230:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789763987.754238:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789763987.754242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763987.754250:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067751552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763987.754293:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061dd500 x1876702067750784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754316:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067750784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754329:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754338:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067750784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763987.754346:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763987.754374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061df480 x1876702067750912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754403:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067750912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754412:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754417:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754420:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067750912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763987.754451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763987.754474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061df100 x1876702067751040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754500:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067751040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763987.754525:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763987.754545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061dd880 x1876702067751168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754571:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067751168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754580:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763987.754608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763987.754629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754649:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061ddc00 x1876702067751296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754662:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067751296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754671:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754675:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754679:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763987.754685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067751424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763987.754707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061de300 x1876702067751424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067751424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754745:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067751424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763987.754754:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067751552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763987.754774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763987.754786:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763987/real 1789763987] req@ffff9360061ddf80 x1876702067751552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763987.754830:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067751552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763987.754857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763987.754862:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763987.754866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067751552:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763988.521941:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763989.545969:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763990.569971:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763991.594015:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763992.617945:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763992.873901:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.873912:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763992.873917:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.873929:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.873942:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.873951:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763992.873997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067751680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763992.874028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.874032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763992.874034:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874038:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874042:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874046:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763992.874058:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763992.874072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067751808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763992.874083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.874088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763992.874090:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874095:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874098:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874104:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874111:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067751680:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789763992.874111:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763992.874129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067751936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763992.874142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.874147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763992.874149:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874165:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763992.874171:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763992.874184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067752064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763992.874196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789763992.874202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.874208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763992.874210:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874217:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874222:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874226:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763992.874232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763992.874244:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067752192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763992.874254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763992.874258:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa6300 x1876702067751680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763992.874260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763992.874264:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874272:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874282:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874290:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763992.874297:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763992.874300:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067751680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874317:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763992.874318:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067752320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763992.874323:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067751680:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789763992.874359:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751808:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763992.874361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763992.874370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763992.874373:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763992.874379:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1460 00000100:00080000:3.0:1789763992.874384:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763992.874391:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789763992.874392:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789763992.874405:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763992.874410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa7800 x1876702067751808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067751808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763992.874437:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067752448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763992.874441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874455:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763992.874501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763992.874517:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa7b80 x1876702067751936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067751936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874537:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763992.874540:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874542:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067751936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763992.874545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763992.874556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa5180 x1876702067752064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874571:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067752064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874576:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763992.874578:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763992.874583:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789763992.874594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa4380 x1876702067752192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067752192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874612:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763992.874614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874616:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763992.874619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067752320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763992.874630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa4a80 x1876702067752320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874643:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067752320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763992.874650:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874652:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067752320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763992.874655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067752448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789763992.874667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789763992.874674:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763992/real 1789763992] req@ffff936010aa5880 x1876702067752448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763992.874680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067752448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763992.874684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763992.874686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789763992.874688:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067752448:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789763993.641995:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763994.666019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763995.689971:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763996.714008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763997.737971:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763997.993918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.993928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763997.993933:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.993943:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.993954:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763997.993963:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789763997.994014:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067752576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994044:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763997.994045:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994051:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763997.994053:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994062:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763997.994072:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067752704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763997.994084:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994089:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763997.994095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067752576:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789763997.994097:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994106:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763997.994114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067752832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763997.994125:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994129:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763997.994132:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994135:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763997.994143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067752960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789763997.994154:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994157:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994162:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763997.994165:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994171:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763997.994184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067753088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763997.994201:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994207:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994210:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763997.994215:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752704:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789763997.994216:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994223:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763997.994236:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067753216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763997.994243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763997.994246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763997.994248:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763997.994251:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1465 00000100:00080000:3.0:1789763997.994254:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763997.994257:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789763997.994261:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763997.994268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789763997.994275:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067753344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763997.994299:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763997.994325:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789763997.994345:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067753216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789763997.994370:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067753344:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789763998.761985:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789763999.785965:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764000.809994:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789764001.066128:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789764001.066175:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764001.066193:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066234:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066246:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066254:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066264:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066274:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764001.066287:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764001.066316:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa4700 x1876702067753344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066349:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067753344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066364:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066371:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067753344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764001.066387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa7100 x1876702067752832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067752832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066420:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066428:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764001.066437:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa4000 x1876702067752960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067752960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066461:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764001.066480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa5c00 x1876702067753088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066494:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067753088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066502:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066506:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764001.066517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa6a00 x1876702067753216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067753216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066545:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067753216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764001.066556:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa5f80 x1876702067752576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067752576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067752576:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764001.066597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763997/real 1789764001] req@ffff936010aa6680 x1876702067752704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764001.066611:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067752704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764001.066619:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764001.066623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764001.066627:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067752704:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764001.833948:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764002.857984:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764003.113895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.113904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764003.113908:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.113919:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.113930:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764003.113937:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764003.113977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067753472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.114005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764003.114007:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114010:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.114012:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764003.114014:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764003.114023:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764003.114033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067753600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.114043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764003.114045:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.114050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764003.114051:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764003.114055:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764003.114060:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067753472:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764003.114066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067753728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.114084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764003.114086:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.114094:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764003.114098:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764003.114104:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764003.114118:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067753856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.114133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764003.114135:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114139:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.114144:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764003.114150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764003.114153:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764003.114161:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764003.114175:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067753984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764003.114189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764003.114190:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114194:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:3.0:1789764003.114198:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764003.114202:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114207:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa5500 x1876702067753472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789764003.114209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764003.114223:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067754112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764003.114233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789764003.114237:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067753472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764003.114239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764003.114244:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764003.114255:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1470 00000100:00080000:0.0:1789764003.114258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764003.114260:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764003.114269:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764003.114269:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114280:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067753472:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789764003.114288:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764003.114290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764003.114306:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067754240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764003.114327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114346:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa4e00 x1876702067753600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067753600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114376:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114379:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114383:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764003.114416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764003.114440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa7480 x1876702067753728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114467:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067753728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114480:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764003.114490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764003.114513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa6680 x1876702067753856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114539:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067753856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114555:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114559:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764003.114571:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764003.114593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114604:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa5f80 x1876702067753984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067753984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067753984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764003.114628:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067754112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764003.114647:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114658:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa6a00 x1876702067754112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114671:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067754112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114679:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067754112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764003.114693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067754240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764003.114715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764003.114744:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764003/real 1789764003] req@ffff936010aa5c00 x1876702067754240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764003.114759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067754240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764003.114767:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764003.114769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764003.114772:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067754240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764003.881951:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764004.906020:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764005.929937:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764006.953967:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764007.977939:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764008.233897:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.233905:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764008.233910:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764008.233920:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.233931:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.233939:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764008.233983:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067754368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764008.234007:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234010:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764008.234011:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764008.234014:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234017:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234019:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234028:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764008.234037:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067754496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764008.234046:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234049:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764008.234051:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764008.234055:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067754368:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764008.234057:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234064:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234067:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234072:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764008.234090:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067754624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764008.234101:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234105:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764008.234107:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764008.234112:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234119:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234123:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234129:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789764008.234130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764008.234144:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067754752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764008.234152:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234155:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764008.234157:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764008.234160:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234163:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234167:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234171:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764008.234181:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067754880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764008.234187:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef6a00 x1876702067754368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764008.234190:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234197:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764008.234200:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764008.234207:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234210:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234213:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764008.234221:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067754368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764008.234227:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067755008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764008.234237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764008.234238:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764008.234244:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764008.234247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764008.234248:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764008.234255:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067754368:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764008.234255:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1476 00000100:00080000:1.0:1789764008.234261:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764008.234265:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764008.234268:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764008.234275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067755136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764008.234286:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764008.234311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234322:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef5f80 x1876702067754496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234331:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067754496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234340:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764008.234346:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764008.234358:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef6680 x1876702067754624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067754624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234377:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234379:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234381:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764008.234384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764008.234396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234403:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef7100 x1876702067754752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067754752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234417:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234419:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764008.234422:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764008.234433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234439:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef5500 x1876702067754880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234446:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067754880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234454:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067754880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764008.234458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764008.234469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234475:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef4e00 x1876702067755008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234482:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067755008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234487:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764008.234503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764008.234515:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764008.234521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764008/real 1789764008] req@ffff936010ef6300 x1876702067755136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764008.234529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067755136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764033 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764008.234534:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764008.234536:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764008.234537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764009.002001:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764010.025947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764011.049989:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764012.073998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764013.098027:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764013.353898:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.353908:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764013.353911:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.353920:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.353929:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.353935:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764013.353970:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067755264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.353998:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354001:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764013.354002:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.354005:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354008:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354010:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354017:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764013.354026:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067755392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.354034:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354037:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764013.354038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.354041:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354046:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354050:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354053:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764013.354067:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067755520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.354075:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354083:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764013.354085:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764013.354086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067755264:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764013.354091:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354103:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354107:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354110:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764013.354118:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067755648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.354125:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354128:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764013.354129:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.354131:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354133:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354135:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354138:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764013.354145:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067755776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.354150:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354154:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764013.354156:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.354158:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354162:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354165:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354168:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764013.354173:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067755904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764013.354181:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764013.354182:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764013.354183:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764013.354186:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1481 00000100:00080000:2.0:1789764013.354188:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764013.354192:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764013.354197:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764013.354198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764013.354211:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067756032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764013.354277:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061dc380 x1876702067755264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354316:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067755264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354330:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354335:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067755264:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764013.354355:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764013.354393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354424:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061dc700 x1876702067755392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354440:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067755392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354452:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764013.354496:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764013.354520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061ddf80 x1876702067755520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067755520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354566:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764013.354583:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764013.354611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061de300 x1876702067755648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354635:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067755648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764013.354673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764013.354693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061ddc00 x1876702067755776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354718:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067755776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354732:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354736:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067755776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764013.354743:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764013.354765:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061dd880 x1876702067755904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354844:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067755904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354856:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067755904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764013.354880:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764013.354919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764013.354934:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764013/real 1789764013] req@ffff9360061df100 x1876702067756032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764013.354947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067756032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764038 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764013.354956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764013.354959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764013.354963:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756032:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764014.121983:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764015.145997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764016.169947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764017.194034:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764018.217978:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764018.473941:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.473949:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764018.473954:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.473963:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.473974:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764018.473982:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764018.474031:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067756160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764018.474064:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764018.474070:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474074:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474078:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764018.474083:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474094:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764018.474108:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067756288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764018.474115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067756160:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764018.474130:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764018.474136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474142:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474145:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764018.474150:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764018.474170:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067756416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764018.474179:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474183:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764018.474185:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474189:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474194:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764018.474199:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764018.474202:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474208:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764018.474218:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067756544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764018.474225:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474227:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764018.474228:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474231:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474234:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764018.474236:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474239:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764018.474247:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067756672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764018.474252:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef6d80 x1876702067756160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764018.474255:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474262:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764018.474267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474275:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474279:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764018.474284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067756160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764018.474285:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474293:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764018.474300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474307:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764018.474310:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067756800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764018.474315:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067756160:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764018.474329:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764018.474332:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764018.474335:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764018.474339:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1486 00000100:00080000:1.0:1789764018.474343:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764018.474347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756288:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764018.474351:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764018.474358:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764018.474369:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067756928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764018.474377:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef7480 x1876702067756288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474407:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067756288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474423:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764018.474428:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764018.474446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474455:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef7b80 x1876702067756416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067756416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474477:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474480:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764018.474485:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764018.474513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474523:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef7800 x1876702067756544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067756544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474544:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474547:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474551:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764018.474557:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764018.474575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef4380 x1876702067756672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067756672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474607:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474610:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067756672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764018.474632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764018.474654:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474664:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef4000 x1876702067756800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474676:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067756800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474687:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764018.474706:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764018.474721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764018.474729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764018/real 1789764018] req@ffff936010ef5c00 x1876702067756928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764018.474740:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067756928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764018.474747:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764018.474750:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764018.474753:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067756928:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764019.241924:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764020.265979:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764021.289999:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764022.314003:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764023.337987:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764023.593908:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.593919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764023.593925:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.593937:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.593949:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764023.593957:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764023.594004:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067757056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764023.594041:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.594046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764023.594048:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.594052:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594055:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764023.594060:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764023.594070:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764023.594087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067757184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764023.594098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.594102:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764023.594104:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.594108:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594112:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764023.594117:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067757056:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789764023.594126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764023.594144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067757312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764023.594158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.594162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764023.594164:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.594168:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594173:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764023.594178:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764023.594184:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764023.594198:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067757440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764023.594208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764023.594211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764023.594216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764023.594219:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.594226:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594233:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764023.594238:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764023.594244:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764023.594268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef4700 x1876702067757056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764023.594270:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067757568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764023.594342:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.594346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764023.594349:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764023.594352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067757056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764023.594357:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594367:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764023.594380:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764023.594380:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789764023.594392:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789764023.594392:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764023.594402:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067757056:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764023.594413:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067757696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764023.594431:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757184:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764023.594456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764023.594461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764023.594463:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764023.594469:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1491 00000100:00080000:1.0:1789764023.594474:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764023.594481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764023.594494:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764023.594504:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764023.594518:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef5180 x1876702067757184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764023.594521:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067757824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764023.594570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067757184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594593:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764023.594625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764023.594644:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594653:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef4a80 x1876702067757312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764023.594662:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067757312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764023.594676:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764023.594688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef5880 x1876702067757440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764023.594707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067757440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594717:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594720:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764023.594730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764023.594752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594765:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef5c00 x1876702067757568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764023.594777:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067757568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067757568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764023.594848:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067757696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764023.594873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594882:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef4000 x1876702067757696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764023.594891:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067757696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067757696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764023.594905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067757824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764023.594924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764023.594935:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764023/real 1789764023] req@ffff936010ef4380 x1876702067757824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764023.594948:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067757824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764023.594957:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764023.594960:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764023.594964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067757824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764024.362014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764025.385941:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764026.410010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764027.433966:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764028.457990:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764028.713941:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.713951:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764028.713957:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.713968:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.713988:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.713997:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764028.714047:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067757952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714073:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714077:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764028.714079:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.714083:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714087:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714090:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764028.714101:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764028.714118:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067758080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714128:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714132:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764028.714134:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.714138:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714142:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714146:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764028.714152:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764028.714163:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067757952:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764028.714165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067758208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714177:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764028.714187:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.714192:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714196:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714199:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764028.714204:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764028.714218:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067758336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714228:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714231:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764028.714233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.714237:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714242:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714245:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789764028.714256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789764028.714258:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764028.714275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067758464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714289:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764028.714291:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764028.714296:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714300:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714304:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764028.714308:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764028.714319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061df480 x1876702067757952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764028.714325:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067758592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764028.714387:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764028.714390:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764028.714393:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764028.714395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067757952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764028.714401:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1496 00000100:00080000:2.0:1789764028.714410:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764028.714418:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789764028.714422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789764028.714425:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764028.714436:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764028.714441:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067758720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764028.714446:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067757952:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764028.714460:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764028.714498:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061dd500 x1876702067758080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714526:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067758080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714551:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714555:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764028.714580:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764028.714599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714611:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061dca80 x1876702067758208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067758208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714630:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714633:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764028.714641:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764028.714662:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061df800 x1876702067758336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067758336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714692:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714695:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714698:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764028.714704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764028.714723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061dfb80 x1876702067758464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067758464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764028.714762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067758592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764028.714782:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714792:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061de680 x1876702067758592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714852:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067758592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714860:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067758592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764028.714872:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067758720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764028.714893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764028.714904:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764028/real 1789764028] req@ffff9360061dc000 x1876702067758720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764028.714915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067758720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764028.714921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764028.714924:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764028.714927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067758720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764029.481933:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764030.505986:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764031.529900:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764032.553992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764033.578014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764033.833953:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.833963:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764033.833968:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.833980:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.833994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834003:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764033.834057:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067758848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764033.834091:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834095:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764033.834097:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.834101:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.834104:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834109:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834120:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764033.834133:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067758976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764033.834140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067758848:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764033.834148:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834153:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764033.834155:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.834161:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.834164:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834168:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834173:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764033.834185:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067759104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764033.834194:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834198:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764033.834200:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.834203:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.834209:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834214:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789764033.834221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764033.834235:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067759232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764033.834246:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764033.834252:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.834256:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.834260:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834264:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834269:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764033.834276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef7800 x1876702067758848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764033.834284:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067759360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764033.834294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067758848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764033.834298:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834307:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764033.834310:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764033.834312:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764033.834316:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:0.0:1789764033.834320:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764033.834322:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764033.834327:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067758848:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764033.834329:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834334:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764033.834348:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067759488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764033.834357:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764033.834361:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789764033.834364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758976:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764033.834365:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764033.834374:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1501 00000100:00080000:1.0:1789764033.834378:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764033.834382:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764033.834387:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764033.834393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764033.834402:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067759616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764033.834410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef7b80 x1876702067758976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834433:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067758976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834442:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834445:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067758976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764033.834452:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764033.834468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764033.834475:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef7480 x1876702067759104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067759104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834494:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834497:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764033.834517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764033.834538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764033.834549:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef6d80 x1876702067759232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834560:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067759232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834568:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834570:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764033.834576:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764033.834596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764033.834607:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef6300 x1876702067759360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834618:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067759360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834626:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834629:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834632:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764033.834639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067759488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764033.834658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764033.834669:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef4e00 x1876702067759488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067759488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067759488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764033.834713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067759616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764033.834731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764033.834739:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764033/real 1789764033] req@ffff936010ef5500 x1876702067759616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764033.834746:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067759616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764033.834751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764033.834753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764033.834755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067759616:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764034.601992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764035.626085:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764036.649956:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764037.673994:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764038.697962:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764038.953959:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.953970:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764038.953974:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764038.953984:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.953994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954000:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764038.954036:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067759744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954066:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.954069:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764038.954071:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764038.954075:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954079:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954083:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954093:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764038.954105:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067759872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954116:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.954120:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764038.954123:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764038.954129:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067759744:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764038.954130:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954138:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954144:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954149:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764038.954161:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067760000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954170:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.954174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764038.954176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764038.954180:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954184:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954189:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954193:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764038.954205:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067760128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954216:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764038.954224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764038.954226:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764038.954231:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764038.954237:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954245:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954249:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954255:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764038.954268:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067760256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954277:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.954280:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789764038.954284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061dce00 x1876702067759744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764038.954285:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764038.954303:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954309:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764038.954316:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067759744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764038.954316:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954335:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764038.954337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954345:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954349:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067759744:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764038.954356:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067760384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764038.954367:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764038.954370:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764038.954372:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764038.954376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759872:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764038.954383:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1506 00000100:00080000:2.0:1789764038.954389:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764038.954393:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764038.954398:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764038.954402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061dd180 x1876702067759872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764038.954422:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067760512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764038.954437:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067759872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954486:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067759872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764038.954493:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764038.954518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954538:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061dc000 x1876702067760000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764038.954550:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067760000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954557:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954560:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954563:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764038.954569:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764038.954587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954596:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061de680 x1876702067760128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764038.954606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067760128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954614:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764038.954632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764038.954650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954659:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061dfb80 x1876702067760256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764038.954669:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067760256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764038.954688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067760384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764038.954705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954713:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061df800 x1876702067760384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764038.954724:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067760384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954731:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954738:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067760384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764038.954756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067760512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764038.954776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764038.954787:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764038/real 1789764038] req@ffff9360061dca80 x1876702067760512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764038.954823:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067760512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764038.954829:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764038.954831:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764038.954833:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067760512:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764039.721939:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764040.745970:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764041.769948:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764042.793978:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764043.818016:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764044.073968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.073980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764044.073983:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764044.073994:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074004:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764044.074011:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764044.074047:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067760640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764044.074089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.074093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764044.074095:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764044.074099:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074103:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764044.074107:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764044.074116:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764044.074129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067760768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764044.074141:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067760640:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764044.074142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.074151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764044.074152:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764044.074157:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074161:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764044.074165:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764044.074170:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764044.074182:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067760896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764044.074191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.074195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764044.074197:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789764044.074200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764044.074204:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074210:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764044.074214:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764044.074220:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764044.074231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067761024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764044.074236:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef7100 x1876702067760640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764044.074252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.074256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1789764044.074258:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067760640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764044.074260:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764044.074269:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074273:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764044.074276:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764044.074277:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789764044.074282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789764044.074285:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764044.074286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067760640:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764044.074303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067761152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764044.074309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760768:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764044.074322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764044.074327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764044.074329:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789764044.074333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764044.074337:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00080000:1.0:1789764044.074342:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764044.074345:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef6680 x1876702067760768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764044.074350:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764044.074359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067760768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789764044.074359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764044.074370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764044.074376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764044.074378:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067761280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764044.074382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760896:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764044.074395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764044.074399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764044.074403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764044.074406:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764044.074409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef5f80 x1876702067760896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764044.074414:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1511 00000100:00100000:0.0:1789764044.074422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067760896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764044.074422:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764044.074443:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789764044.074444:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074449:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789764044.074450:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764044.074452:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067760896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764044.074458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764044.074476:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067761408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764044.074479:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764044.074489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef6a00 x1876702067761024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764044.074502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067761024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764044.074510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764044.074518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764044.074533:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764044.074551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764044.074558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef5500 x1876702067761152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764044.074565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067761152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764044.074570:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074572:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764044.074573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764044.074577:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067761280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764044.074587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764044.074593:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef4e00 x1876702067761280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764044.074599:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067761280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764044.074603:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764044.074606:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067761280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764044.074609:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067761408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764044.074618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764044.074623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764044/real 1789764044] req@ffff936010ef6300 x1876702067761408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764044.074628:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067761408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764044.074632:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764044.074634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764044.074635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067761408:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764044.841951:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764045.865983:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764046.889946:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764047.914069:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764048.937934:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764049.193905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.193914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764049.193917:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.193925:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.193933:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.193938:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764049.193968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067761536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764049.193990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.193992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764049.193994:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.193996:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.193998:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194000:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194007:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764049.194014:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067761664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764049.194026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.194029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764049.194030:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.194034:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.194036:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194039:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194042:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764049.194048:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067761792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764049.194054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.194056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764049.194057:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.194060:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.194062:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194065:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194067:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764049.194076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067761920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764049.194082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067761536:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764049.194084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.194090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764049.194097:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.194102:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.194103:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194105:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764049.194115:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067762048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764049.194119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.194121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764049.194122:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.194123:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.194125:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194127:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194131:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764049.194140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067762176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764049.194150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764049.194154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764049.194155:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764049.194159:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1517 00000100:00080000:1.0:1789764049.194167:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764049.194171:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764049.194175:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764049.194194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067762304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764049.194205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef6d80 x1876702067761536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194307:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067761536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194319:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194324:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067761536:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764049.194351:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764049.194382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef7480 x1876702067761664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067761664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194440:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764049.194454:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764049.194489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194529:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef7b80 x1876702067761792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194544:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067761792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194552:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194556:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194559:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764049.194591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764049.194618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194629:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef7800 x1876702067761920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067761920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194658:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067761920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764049.194664:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764049.194682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef4380 x1876702067762048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194710:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067762048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194725:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764049.194730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067762176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764049.194750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.194761:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef4000 x1876702067762176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.194771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067762176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.194778:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.194781:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.194783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067762176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764049.194789:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067762304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764049.195343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764049.195358:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764049/real 1789764049] req@ffff936010ef5c00 x1876702067762304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764049.195369:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067762304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764049.195382:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764049.195386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764049.195404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067762304:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764049.962085:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764050.985937:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764052.010025:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764053.033971:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764054.057929:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764054.313931:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764054.313941:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764054.313947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764054.313958:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00080000:1.0:1789764054.313971:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764054.313980:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764054.314029:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067762432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764054.314063:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764054.314067:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764054.314069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764054.314073:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00080000:1.0:1789764054.314079:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764054.314083:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764054.314094:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764054.314115:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067762560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764054.314129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764054.314132:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067762432:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764054.314134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764054.314137:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764054.314154:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00080000:1.0:1789764054.314160:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764054.314165:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764054.314172:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764054.314187:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067762688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764054.314199:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764054.314203:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764054.314205:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764054.314209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764054.314215:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00080000:1.0:1789764054.314222:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764054.314230:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764054.314236:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764054.314253:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef5880 x1876702067762432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764054.314258:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067762816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764054.314283:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067762432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764054.314322:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764054.314329:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067762432:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764054.314346:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764054.314364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762560:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764054.314380:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764054.314384:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764054.314391:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1522 00000800:00000100:2.0:1789764054.314399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764054.314402:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764054.314410:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764054.314419:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764054.314423:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef4a80 x1876702067762560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764054.314440:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067762944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764054.314443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067762560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764054.314477:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764054.314485:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764054.314487:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764054.314492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764054.314493:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764054.314498:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762560:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764054.314505:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00080000:1.0:1789764054.314512:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764054.314519:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762688:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764054.314523:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764054.314536:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764054.314552:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067763072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764054.314571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764054.314588:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764054.314589:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef5180 x1876702067762688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764054.314604:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764054.314610:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764054.314622:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1522 00000100:00100000:2.0:1789764054.314624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067762688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764054.314633:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764054.314635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764054.314643:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764054.314648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762688:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764054.314653:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764054.314664:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764054.314675:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067763200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764054.314695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764054.314709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef4700 x1876702067762816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764054.314719:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067762816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764054.314725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764054.314730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764054.314742:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764054.314755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764054.314761:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef5880 x1876702067762944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764054.314776:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067762944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764054.314780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764054.314787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067762944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764054.314790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764054.314822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764054.314829:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef5c00 x1876702067763072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764054.314837:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067763072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764054.314841:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314843:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764054.314845:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764054.314848:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764054.314857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764054.314863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764054/real 1789764054] req@ffff936010ef4000 x1876702067763200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764054.314869:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067763200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764079 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764054.314877:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764054.314878:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764054.314880:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763200:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764055.081966:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764056.105985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764057.130049:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764058.154012:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764059.177997:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764059.433948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.433959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764059.433963:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.433973:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.433982:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.433988:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764059.434022:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067763328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764059.434047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764059.434052:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434056:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434059:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434062:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434071:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764059.434081:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067763456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764059.434092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764059.434098:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434106:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434109:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434114:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764059.434125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067763584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764059.434138:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067763328:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764059.434141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764059.434158:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434162:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434165:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434170:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434174:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764059.434185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067763712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764059.434193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764059.434198:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434202:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434221:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434225:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434230:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764059.434243:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764059.434249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067763840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764059.434260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764059.434264:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434268:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434271:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434275:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434279:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764059.434292:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067763968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764059.434302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764059.434305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764059.434307:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764059.434312:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1527 00000100:00080000:3.0:1789764059.434332:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764059.434336:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764059.434340:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764059.434350:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067764096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764059.434400:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa4000 x1876702067763328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.434425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067763328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.434437:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.434598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.434603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067763328:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764059.434613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764059.434643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.434657:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa7100 x1876702067763456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.434669:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067763456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.434678:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.434681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.434685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764059.434719:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764059.434740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.434752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa4700 x1876702067763584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.434764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067763584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.434772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.434776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.434779:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764059.434786:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764059.434911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.434924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa5880 x1876702067763712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.434936:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067763712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.434944:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.434947:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.434951:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764059.434957:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764059.434977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.434989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa4a80 x1876702067763840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.435001:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067763840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.435008:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.435011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.435014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067763840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764059.435021:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764059.435040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.435050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa4380 x1876702067763968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.435062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067763968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.435070:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.435073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.435076:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067763968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764059.435082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764059.435102:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764059.435114:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764059/real 1789764059] req@ffff936010aa5180 x1876702067764096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764059.435126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067764096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764059.435141:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764059.435145:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764059.435149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764096:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764060.202275:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764061.225978:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764062.249985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764063.273971:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764064.297954:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764064.553909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.553919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764064.553922:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.553932:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.553942:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.553948:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764064.553982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067764224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764064.554007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764064.554012:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554016:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.554019:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.554024:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554033:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764064.554042:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067764224:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764064.554045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067764352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764064.554056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764064.554061:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.554078:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.554082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764064.554098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067764480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764064.554107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764064.554112:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554115:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.554119:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.554122:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554127:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764064.554140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067764608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764064.554150:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764352:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764064.554152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764064.554160:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554164:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.554167:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.554171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764064.554190:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067764736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764064.554194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764480:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764064.554206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764064.554212:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554218:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00080000:0.0:1789764064.554222:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764064.554226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764608:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764064.554228:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554236:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764064.554247:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764064.554251:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067764864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764064.554262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764064.554265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764064.554268:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764064.554272:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1532 00000100:00100000:2.0:1789764064.554274:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764864:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764064.554289:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764064.554295:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764064.554300:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764064.554313:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067764992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764064.554349:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764992:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764065.322079:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764066.346693:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764067.370021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789764067.626246:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789764067.626309:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789764067.626329:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764067.627389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020dac380 x1876702067764224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764089 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.627422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067764224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764089 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.627436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.627441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.627445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067764224:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764067.627639:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764067.628367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020daf480 x1876702067764352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789764067.628371:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789764067.628389:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764067.628395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067764352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789764067.628404:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:2.0:1789764067.628415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:3.0:1789764067.628420:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:2.0:1789764067.628424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764067.628436:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764067.628444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020dae680 x1876702067764480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.628460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067764480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.628468:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.628472:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764067.628492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020dad180 x1876702067764992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.628510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067764992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.628517:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.628521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764067.628531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020dae300 x1876702067764864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.628543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067764864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.628550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.628553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628556:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067764864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764067.628563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020dadf80 x1876702067764608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.628575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067764608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.628582:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.628586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764067.628595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764064/real 1789764067] req@ffff936020daf800 x1876702067764736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764067.628605:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067764736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764067.628612:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764067.628615:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764067.628618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067764736:192.168.203.137@tcp:8:kworker.0 00010000:00010000:3.0:1789764068.393950:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764069.418013:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764069.674058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764069.674069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764069.674074:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.674086:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.674099:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.674109:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764069.674154:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067765120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764069.674185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764069.674189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764069.674191:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.674195:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.674199:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.674204:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764069.674215:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764069.674228:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067765248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764069.674621:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067765120:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764069.674759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764069.674765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764069.674767:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.674772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1537 00000800:00000100:2.0:1789764069.674839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764069.675287:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.675295:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764069.675303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764069.675322:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067765376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764069.675333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020daea00 x1876702067765120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764069.675337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764069.675346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764069.675350:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.675356:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.675360:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.675366:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764069.675366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067765120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675383:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675387:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067765120:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764069.675395:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764069.675420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764069.675431:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020dace00 x1876702067765248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764069.675442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067765248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675449:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675455:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765248:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789764069.675477:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764069.675481:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764069.675499:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067765504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764069.675504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764069.675516:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764069.675520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020dadc00 x1876702067765376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764069.675522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764069.675525:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.675533:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.675539:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.675547:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764069.675554:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764069.675555:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067765376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675567:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789764069.675570:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067765632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764069.675573:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765376:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764069.675596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764069.675596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765504:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764069.675606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764069.675608:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.675612:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.675616:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.675620:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764069.675625:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764069.675627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764069.675641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067765760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764069.675643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020dace00 x1876702067765504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764069.675665:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764069.675671:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067765504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764069.675672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764069.675675:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764069.675684:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1537 00000100:00080000:0.0:1789764069.675688:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764069.675691:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764069.675696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764069.675706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067765888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764069.675735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675743:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764069.675755:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764069.675774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764069.675784:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020daea00 x1876702067765632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764069.675794:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067765632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675825:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675829:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675832:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067765632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764069.675837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067765760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764069.675854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764069.675863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020daf800 x1876702067765760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764069.675873:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067765760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675879:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675882:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675885:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067765760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764069.675890:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067765888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764069.675903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764069.675911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764069/real 1789764069] req@ffff936020dadf80 x1876702067765888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764069.675920:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067765888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764069.675927:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764069.675929:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764069.675932:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067765888:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764070.442013:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764071.465995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764072.490012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764073.514026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764074.538005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764074.793933:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.793943:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764074.793947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.793958:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.793971:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.793978:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764074.794029:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067766016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794063:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794067:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764074.794069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794073:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794077:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794081:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764074.794091:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764074.794103:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067766144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794112:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794115:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764074.794116:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794120:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794124:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794127:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764074.794131:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764074.794140:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067766272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794149:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794152:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764074.794153:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794157:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794161:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794164:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764074.794168:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764074.794178:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067766400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794186:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794189:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764074.794191:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794194:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794197:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794259:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067766016:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789764074.794265:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764074.794277:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067766528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764074.794290:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794292:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794295:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794298:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764074.794302:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764074.794309:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067766656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764074.794317:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764074.794320:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764074.794321:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764074.794324:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1542 00000100:00080000:0.0:1789764074.794327:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764074.794330:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764074.794333:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764074.794335:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764074.794340:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067766784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764074.794383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020dae300 x1876702067766016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794403:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067766016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067766016:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764074.794430:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764074.794454:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020dad180 x1876702067766144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794480:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067766144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794488:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794491:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764074.794499:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764074.794520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794532:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020dae680 x1876702067766272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794544:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067766272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794552:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794555:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794559:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764074.794564:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764074.794584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020daf480 x1876702067766400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067766400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794614:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764074.794648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764074.794669:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020dac380 x1876702067766528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067766528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067766528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764074.794711:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067766656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764074.794729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794740:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020daf100 x1876702067766656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067766656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794759:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067766656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764074.794771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067766784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764074.794790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764074.794823:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764074/real 1789764074] req@ffff936020dac700 x1876702067766784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764074.794836:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067766784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764074.794843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764074.794846:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764074.794850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067766784:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764075.562255:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764076.586014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764077.609929:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764078.633996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764079.658051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764079.914025:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914034:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764079.914038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764079.914047:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1547 00000100:00080000:3.0:1789764079.914059:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914065:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764079.914115:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067766912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764079.914143:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914148:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764079.914150:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764079.914175:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1547 00000100:00080000:3.0:1789764079.914181:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914186:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914207:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764079.914222:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067767040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764079.914224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067766912:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764079.914240:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914243:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764079.914245:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764079.914249:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1547 00000100:00080000:3.0:1789764079.914253:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914257:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914263:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764079.914280:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067767168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764079.914297:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914300:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764079.914302:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764079.914309:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1547 00000800:00000100:2.0:1789764079.914311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764079.914317:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914325:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914332:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764079.914349:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067767296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764079.914356:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914359:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764079.914362:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764079.914365:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa7b80 x1876702067766912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764079.914366:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1547 00000100:00080000:3.0:1789764079.914374:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914388:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914398:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764079.914405:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067766912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764079.914419:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067767424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764079.914422:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764079.914436:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764079.914437:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764079.914449:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764079.914455:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764079.914456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067766912:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764079.914465:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1547 00000100:00080000:3.0:1789764079.914474:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914478:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914484:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764079.914495:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067767552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764079.914501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767040:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764079.914521:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764079.914525:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764079.914528:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764079.914534:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1547 00000800:00000100:2.0:1789764079.914541:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764079.914541:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764079.914550:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764079.914556:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764079.914563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa7800 x1876702067767040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764079.914572:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067767680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764079.914603:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067767040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914618:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764079.914630:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764079.914649:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764079.914660:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa6300 x1876702067767168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764079.914670:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067767168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914678:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764079.914690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764079.914705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764079.914714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa5180 x1876702067767296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764079.914724:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067767296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914731:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914738:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764079.914743:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764079.914758:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764079.914767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa4380 x1876702067767424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764079.914777:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067767424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914792:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764079.914837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067767552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764079.914855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764079.914865:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa4a80 x1876702067767552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764079.914876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067767552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067767552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764079.914895:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067767680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764079.914910:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764079.914918:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764079/real 1789764079] req@ffff936010aa5880 x1876702067767680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764079.914928:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067767680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764079.914935:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764079.914938:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764079.914942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067767680:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764080.682272:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764081.706168:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764082.731217:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764083.754034:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764084.778429:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764085.034017:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034028:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764085.034034:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034046:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034060:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034069:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764085.034117:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067767808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034233:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034237:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764085.034239:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034244:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034248:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034253:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034264:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764085.034280:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067767936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034290:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034294:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764085.034296:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034300:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034303:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034307:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034311:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764085.034322:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067768064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034331:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034335:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764085.034336:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034340:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034344:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034348:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034353:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764085.034363:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067768192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034372:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034376:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764085.034378:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034382:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034385:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034389:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034394:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764085.034404:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067768320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034414:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034417:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764085.034418:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034422:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034425:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034429:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034433:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764085.034444:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067768448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764085.034452:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764085.034455:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764085.034457:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764085.034461:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1552 00000100:00080000:3.0:1789764085.034464:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764085.034468:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764085.034472:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764085.034483:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067768576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764085.034820:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067767808:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764085.034896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.034941:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa4700 x1876702067767808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.034962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067767808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.034972:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.034976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.034980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067767808:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764085.034988:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764085.035012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035026:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa7100 x1876702067767936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067767936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067767936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764085.035058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764085.035078:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa4000 x1876702067768064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067768064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035110:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035112:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764085.035122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764085.035477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035491:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa5c00 x1876702067768192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067768192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764085.035524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764085.035544:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035556:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa6a00 x1876702067768320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035567:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067768320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035578:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035582:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764085.035588:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067768448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764085.035609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa5f80 x1876702067768448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035632:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067768448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035643:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035646:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067768448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764085.035652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067768576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764085.035671:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764085.035683:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764085/real 1789764085] req@ffff936010aa6680 x1876702067768576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764085.035694:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067768576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764085.035702:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764085.035705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764085.035709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067768576:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764085.801986:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764086.825967:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764087.850062:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764088.874220:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764089.898012:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764090.153934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.153945:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764090.153949:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.153959:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.153968:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764090.153975:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764090.154009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067768704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764090.154039:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154043:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154046:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764090.154050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154059:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764090.154070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067768832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764090.154085:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764090.154110:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067768704:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764090.154114:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154129:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764090.154144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067768960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764090.154159:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154163:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154166:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764090.154170:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154174:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764090.154184:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067769088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764090.154199:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154203:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154207:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789764090.154212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764090.154217:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154224:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764090.154238:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067769216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764090.154254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154258:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154263:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764090.154268:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154273:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764090.154279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef4380 x1876702067768704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764090.154292:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067769344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764090.154302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764090.154310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764090.154312:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764090.154317:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1557 00000100:00080000:1.0:1789764090.154322:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764090.154326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067768704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764090.154331:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764090.154339:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789764090.154353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154367:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067768704:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764090.154395:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764090.154405:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067769472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764090.154427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef7800 x1876702067768832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067768832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154472:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764090.154482:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764090.154502:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154514:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef7b80 x1876702067768960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154541:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067768960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154556:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067768960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764090.154563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764090.154584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef7480 x1876702067769088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067769088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154618:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154621:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154624:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764090.154630:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764090.154645:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef6d80 x1876702067769216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154664:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067769216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154671:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154674:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764090.154683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067769344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764090.154700:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154710:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef6300 x1876702067769344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067769344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154728:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154732:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067769344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764090.154754:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067769472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764090.154772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764090.154781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764090/real 1789764090] req@ffff936010ef4e00 x1876702067769472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764090.154793:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067769472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764090.154826:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764090.154829:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764090.154833:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067769472:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764090.922527:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764091.946016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764092.969954:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764093.993997:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764095.017969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764095.273930:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.273939:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764095.273942:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.273952:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.273962:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.273968:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764095.274005:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067769600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764095.274030:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.274032:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764095.274033:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274036:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.274039:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274041:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764095.274069:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764095.274084:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067769728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764095.274095:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764095.274098:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067769600:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764095.274100:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764095.274104:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274110:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.274113:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274115:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764095.274119:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764095.274128:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067769856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764095.274134:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.274136:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764095.274137:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274140:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.274149:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274152:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764095.274155:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764095.274164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067769984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764095.274165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764095.274172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.274174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764095.274176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274178:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.274180:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274182:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764095.274185:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764095.274190:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067770112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764095.274195:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.274197:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764095.274198:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274200:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00080000:3.0:1789764095.274203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274205:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764095.274207:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764095.274212:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067770240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764095.274219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa7480 x1876702067769600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764095.274223:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764095.274232:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764095.274236:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764095.274242:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1563 00000100:00100000:2.0:1789764095.274247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067769600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764095.274249:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764095.274257:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764095.274260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274264:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789764095.274264:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764095.274270:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067769600:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764095.274279:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067770368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764095.274293:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764095.274318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa4e00 x1876702067769728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274339:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067769728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274344:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274347:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274350:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764095.274356:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764095.274374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274385:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa5500 x1876702067769856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067769856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274407:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764095.274416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764095.274435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274442:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa6680 x1876702067769984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274449:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067769984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274454:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274456:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274458:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067769984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764095.274461:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764095.274472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa5f80 x1876702067770112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067770112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274488:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274490:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274491:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764095.274494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067770240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764095.274504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa6a00 x1876702067770240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274523:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067770240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274531:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067770240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764095.274535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067770368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764095.274545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764095.274551:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764095/real 1789764095] req@ffff936010aa5c00 x1876702067770368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764095.274557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067770368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764120 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764095.274561:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764095.274563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764095.274565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067770368:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764096.041988:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764097.066055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764098.089998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764099.113956:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764100.138109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764100.393962:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.393974:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764100.393978:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.393989:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394003:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394011:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764100.394069:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067770496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764100.394095:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394097:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764100.394099:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394102:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394105:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394108:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764100.394117:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764100.394128:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067770624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764100.394135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394137:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764100.394139:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394141:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394143:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394147:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067770496:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789764100.394150:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764100.394161:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067770752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764100.394168:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394171:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764100.394172:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394174:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394176:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394178:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764100.394180:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764100.394188:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067770880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764100.394193:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394196:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764100.394197:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394200:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394203:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394205:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764100.394208:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764100.394217:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067771008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764100.394223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764100.394229:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394231:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764100.394232:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394236:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394238:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764100.394240:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764100.394244:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764100.394256:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067771136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764100.394265:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764100.394268:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764100.394270:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764100.394275:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1568 00000100:00080000:1.0:1789764100.394280:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764100.394282:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef5500 x1876702067770496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764100.394287:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764100.394294:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764100.394306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067771264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764100.394307:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067770496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394332:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394336:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394339:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067770496:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764100.394345:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764100.394362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394371:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef6a00 x1876702067770624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067770624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394384:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394388:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764100.394407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764100.394417:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394424:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef5f80 x1876702067770752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394431:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067770752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394437:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394439:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764100.394442:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764100.394453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef6680 x1876702067770880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067770880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394470:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394472:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067770880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764100.394475:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764100.394484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394490:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef7100 x1876702067771008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067771008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764100.394507:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067771136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764100.394517:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef4e00 x1876702067771136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067771136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394532:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067771136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764100.394540:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067771264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764100.394557:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764100.394567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764100/real 1789764100] req@ffff936010ef6300 x1876702067771264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764100.394579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067771264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764125 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764100.394587:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764100.394590:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764100.394593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067771264:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764101.161939:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764102.186014:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764103.210006:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764104.234024:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764105.257947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764105.513914:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.513925:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764105.513930:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.513941:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.513954:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.513969:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764105.514025:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067771392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764105.514050:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514053:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764105.514054:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.514057:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.514060:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.514063:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514072:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764105.514081:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067771520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764105.514089:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514092:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764105.514094:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.514100:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00100000:2.0:1789764105.514102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067771392:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764105.514104:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.514112:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514117:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764105.514130:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067771648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764105.514139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514143:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764105.514145:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.514149:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.514153:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.514168:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514174:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764105.514177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764105.514188:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067771776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764105.514196:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514198:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764105.514199:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.514202:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.514205:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.514207:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764105.514219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067771904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764105.514224:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef6d80 x1876702067771392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764105.514227:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514233:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764105.514236:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764105.514243:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.514246:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764105.514248:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514252:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764105.514255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067771392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764105.514262:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067772032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764105.514272:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764105.514273:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764105.514279:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764105.514281:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764105.514282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764105.514285:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1573 00000100:00080000:1.0:1789764105.514288:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764105.514291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067771392:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764105.514293:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764105.514299:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764105.514306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067772160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764105.514335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764105.514357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514368:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef7480 x1876702067771520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514380:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067771520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514389:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514392:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764105.514401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764105.514416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514426:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef7b80 x1876702067771648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067771648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514444:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514447:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514450:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764105.514455:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764105.514470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514479:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef7800 x1876702067771776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514490:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067771776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764105.514509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764105.514523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514532:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef4380 x1876702067771904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067771904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514549:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514555:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067771904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764105.514561:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764105.514575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514583:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef4000 x1876702067772032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067772032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514601:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764105.514612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764105.514627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764105.514636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764105/real 1789764105] req@ffff936010ef5c00 x1876702067772160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764105.514646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067772160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764105.514653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764105.514656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764105.514660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772160:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764106.281965:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764107.305981:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764108.330004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764109.354148:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764110.377986:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764110.633958:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.633969:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764110.633973:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.633983:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:2.0:1789764110.633994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764110.634001:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764110.634036:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067772288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764110.634072:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634076:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764110.634082:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.634086:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:2.0:1789764110.634090:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764110.634094:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764110.634104:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764110.634114:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067772288:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764110.634119:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067772416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764110.634131:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634135:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764110.634136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.634140:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:2.0:1789764110.634144:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764110.634149:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764110.634154:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764110.634165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067772544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764110.634173:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764110.634178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.634183:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1578 00000800:00000100:0.0:1789764110.634186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764110.634189:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764110.634196:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764110.634201:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764110.634214:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067772672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764110.634223:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764110.634227:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.634230:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:2.0:1789764110.634234:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764110.634237:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634242:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061dd500 x1876702067772288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789764110.634243:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764110.634260:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067772800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764110.634266:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067772288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764110.634272:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634281:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764110.634283:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764110.634284:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764110.634290:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:0.0:1789764110.634291:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789764110.634296:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764110.634297:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067772288:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764110.634306:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764110.634313:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764110.634325:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067772928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764110.634334:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772416:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764110.634337:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764110.634343:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764110.634345:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764110.634351:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1578 00000100:00080000:2.0:1789764110.634355:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764110.634365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764110.634367:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764110.634374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764110.634382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061df480 x1876702067772416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764110.634387:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067773056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764110.634402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067772416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634409:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634412:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764110.634419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764110.634433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061df100 x1876702067772544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764110.634453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067772544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634458:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764110.634465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764110.634476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634482:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061dd880 x1876702067772672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764110.634492:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067772672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634498:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634499:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764110.634502:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764110.634518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061ddc00 x1876702067772800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764110.634530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067772800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634537:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634539:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067772800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764110.634548:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764110.634558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061de300 x1876702067772928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764110.634574:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067772928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634582:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634585:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634586:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067772928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764110.634589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764110.634599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764110.634605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764110/real 1789764110] req@ffff9360061ddf80 x1876702067773056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764110.634611:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067773056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764110.634615:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764110.634617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764110.634618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773056:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764111.402072:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764112.425935:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764113.450009:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764114.473985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764115.498036:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764115.753974:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.753984:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764115.753988:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754000:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754014:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.754023:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764115.754069:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067773184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764115.754319:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067773184:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764115.754402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764115.754438:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.754443:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764115.754445:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754449:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754454:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.754458:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.754469:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764115.754481:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067773312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764115.754544:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061dc700 x1876702067773184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.754564:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067773184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.754574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.754578:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.754583:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067773184:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764115.754606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764115.754629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764115.754642:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061dc380 x1876702067773312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.754652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067773312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.754659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.754662:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.754666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773312:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764115.754776:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.754780:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764115.754785:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754790:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754794:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.754819:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.754825:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764115.754838:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067773440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764115.754852:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.754855:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764115.754857:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754860:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754863:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.754867:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.754873:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764115.754881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764115.754883:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067773568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764115.754899:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.754913:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764115.754915:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754919:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754923:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.754927:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.754931:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764115.754934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764115.754945:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067773696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764115.754954:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061dc380 x1876702067773440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764115.754956:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.754964:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764115.754968:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764115.754974:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.754977:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764115.754981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067773440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764115.754984:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.754990:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764115.754999:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789764115.755004:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067773824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764115.755006:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.755015:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773440:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764115.755028:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764115.755032:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764115.755034:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764115.755036:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773568:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789764115.755040:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1583 00000100:00080000:2.0:1789764115.755046:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764115.755049:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764115.755054:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764115.755064:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067773952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764115.755070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764115.755090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061dc700 x1876702067773568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.755103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067773568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.755111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.755114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.755118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764115.755124:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764115.755142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764115.755152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061ddf80 x1876702067773696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.755163:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067773696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.755171:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.755174:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.755177:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067773696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764115.755183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764115.755200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764115.755210:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061de300 x1876702067773824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.755221:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067773824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.755229:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.755232:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.755235:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764115.755256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764115.755276:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764115.755286:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764115/real 1789764115] req@ffff9360061ddc00 x1876702067773952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764115.755298:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067773952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764115.755306:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764115.755309:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764115.755312:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067773952:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764116.522028:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764117.546009:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764118.569970:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764119.593935:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764120.617941:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764120.873904:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.873914:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764120.873917:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764120.873925:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.873932:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764120.873938:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764120.873969:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067774080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.873988:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.873990:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764120.873991:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764120.873994:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.873996:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764120.873998:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874006:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764120.874015:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067774208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.874022:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.874024:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764120.874025:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764120.874028:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.874029:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764120.874032:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874035:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764120.874043:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067774080:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764120.874045:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067774336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.874056:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.874058:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764120.874059:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764120.874062:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.874065:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764120.874068:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874073:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764120.874088:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067774464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.874098:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.874102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764120.874104:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764120.874109:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.874113:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764120.874117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764120.874120:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874127:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764120.874141:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067774592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.874152:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764120.874156:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764120.874158:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764120.874162:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061dd880 x1876702067774080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764120.874165:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.874170:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764120.874176:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874182:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764120.874189:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067774080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764120.874193:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067774720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764120.874215:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764120.874217:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764120.874218:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764120.874222:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764120.874226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789764120.874228:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1588 00000100:00080000:2.0:1789764120.874233:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764120.874236:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067774080:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764120.874237:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764120.874242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764120.874249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764120.874253:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067774848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764120.874280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874296:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061df100 x1876702067774208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874309:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067774208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874314:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874318:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764120.874344:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764120.874362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874377:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061df480 x1876702067774336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067774336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874410:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874415:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764120.874423:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764120.874447:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061dd500 x1876702067774464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874498:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067774464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874516:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764120.874537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764120.874555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061dca80 x1876702067774592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067774592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874579:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067774592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764120.874582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067774720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764120.874591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061df800 x1876702067774720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067774720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067774720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764120.874625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067774848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764120.874643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764120.874652:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764120/real 1789764120] req@ffff9360061dfb80 x1876702067774848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764120.874659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067774848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764120.874664:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764120.874666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764120.874667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067774848:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764121.642000:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764122.665946:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764123.689994:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764124.713943:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764125.738001:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764125.993977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.993986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764125.993990:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.993999:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994010:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994017:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764125.994063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067774976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764125.994083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764125.994089:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.994092:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994096:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994106:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764125.994118:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764125.994132:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067775104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764125.994143:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067774976:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764125.994146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764125.994156:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.994160:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994164:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994169:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764125.994174:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764125.994188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067775232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764125.994199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764125.994204:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.994208:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994212:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994216:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764125.994220:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789764125.994224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764125.994244:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067775360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764125.994257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764125.994263:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.994267:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994270:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994274:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994278:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef5880 x1876702067774976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789764125.994282:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764125.994299:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067775488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764125.994306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067774976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764125.994315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764125.994324:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764125.994324:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764125.994330:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994334:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764125.994342:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764125.994342:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764125.994351:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764125.994352:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067774976:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764125.994367:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067775616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764125.994378:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764125.994381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764125.994383:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764125.994387:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775104:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764125.994391:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1593 00000100:00080000:1.0:1789764125.994397:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764125.994401:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764125.994407:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789764125.994421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764125.994422:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067775744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764125.994440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef4700 x1876702067775104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067775104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994465:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994469:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994472:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764125.994478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764125.994496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef5180 x1876702067775232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067775232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764125.994540:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764125.994568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994580:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef4a80 x1876702067775360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067775360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764125.994613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764125.994632:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef5c00 x1876702067775488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067775488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994663:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994667:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067775488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764125.994676:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067775616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764125.994696:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994706:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef4000 x1876702067775616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994718:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067775616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994726:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994729:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994732:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067775616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764125.994749:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067775744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764125.994778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764125.994789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764125/real 1789764125] req@ffff936010ef4380 x1876702067775744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764125.994843:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067775744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764125.994851:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764125.994853:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764125.994855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067775744:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764126.762016:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764127.786010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764128.810026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764129.833995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764130.858014:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764131.113946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.113957:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764131.113962:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.113974:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.113989:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.113997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764131.114044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067775872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764131.114085:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.114096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114108:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764131.114119:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067775872:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789764131.114119:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764131.114135:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067776000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764131.114159:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.114163:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114182:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114187:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764131.114192:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764131.114204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067776128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764131.114222:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.114226:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114230:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114234:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764131.114239:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764131.114244:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764131.114254:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067776256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764131.114269:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.114273:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114280:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114284:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764131.114290:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764131.114294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764131.114306:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067776384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764131.114322:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764131.114326:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776256:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764131.114328:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114335:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114341:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764131.114347:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764131.114357:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764131.114363:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067776512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764131.114389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764131.114393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764131.114395:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764131.114399:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1598 00000100:00080000:1.0:1789764131.114404:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764131.114408:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764131.114412:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067776512:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764131.114415:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764131.114433:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067776640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764131.114455:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067776640:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764131.881962:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764132.906025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764133.929936:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764134.186132:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764134.186186:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764134.186206:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186257:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186268:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186277:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186285:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186293:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764134.186301:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764134.186392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef7100 x1876702067776640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067776640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186434:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186439:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186444:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067776640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764134.186452:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef4e00 x1876702067776512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067776512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186477:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186480:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067776512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764134.186487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef6d80 x1876702067776256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186498:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067776256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186506:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186510:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186513:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764134.186520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef6300 x1876702067776384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067776384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186537:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186540:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764134.186550:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef7b80 x1876702067776000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186562:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067776000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186570:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186573:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186576:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764134.186582:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef7480 x1876702067776128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067776128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186601:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764134.186613:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764131/real 1789764134] req@ffff936010ef7800 x1876702067775872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764134.186626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067775872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764134.186633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764134.186636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764134.186684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067775872:192.168.203.137@tcp:250:kworker.0 00010000:00010000:3.0:1789764134.953985:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764135.977961:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764136.233941:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.233951:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764136.233957:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.233969:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.233982:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.233990:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764136.234031:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067776768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234062:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234066:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764136.234068:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.234072:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234076:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.234081:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764136.234093:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764136.234107:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067776768:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764136.234108:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067776896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234124:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234128:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764136.234129:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.234134:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234137:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.234141:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764136.234145:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764136.234159:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067777024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234168:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234172:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764136.234174:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.234177:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234182:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.234185:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764136.234190:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764136.234194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764136.234206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067777152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234217:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234220:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764136.234223:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.234227:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234232:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.234236:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764136.234241:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764136.234253:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020dad880 x1876702067776768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764136.234255:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067777280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234280:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234283:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764136.234285:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764136.234290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067776768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764136.234293:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234301:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764136.234306:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764136.234308:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764136.234313:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:0.0:1789764136.234316:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764136.234318:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067776768:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764136.234334:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067777408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764136.234344:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764136.234348:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789764136.234351:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776896:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764136.234352:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764136.234364:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1604 00000100:00080000:0.0:1789764136.234369:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764136.234374:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764136.234379:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764136.234380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764136.234398:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067777536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764136.234398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020dad500 x1876702067776896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067776896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234443:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234447:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234451:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067776896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764136.234457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764136.234477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764136.234488:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020daed80 x1876702067777024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234500:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067777024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234508:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234511:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764136.234520:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764136.234537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764136.234546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020dac000 x1876702067777152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067777152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234570:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234574:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234577:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764136.234582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764136.234601:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764136.234612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020dafb80 x1876702067777280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234623:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067777280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764136.234643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067777408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764136.234664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764136.234675:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020dac700 x1876702067777408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234686:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067777408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234693:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234696:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234699:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067777408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764136.234718:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067777536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764136.234736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764136.234747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764136/real 1789764136] req@ffff936020daf100 x1876702067777536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764136.234774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067777536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764161 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764136.234783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764136.234786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764136.234790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067777536:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764137.002234:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764138.025936:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764139.049991:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764140.073947:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764141.097971:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764141.353936:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.353944:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764141.353947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.353957:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.353968:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764141.353974:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764141.354013:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067777664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764141.354037:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354040:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764141.354041:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354044:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354047:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764141.354049:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764141.354058:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764141.354065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067777792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764141.354072:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354075:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764141.354077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354082:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354083:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764141.354086:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764141.354089:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764141.354096:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067777920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764141.354103:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354106:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764141.354107:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354111:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354116:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764141.354117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067777664:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764141.354121:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764141.354130:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764141.354142:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067778048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764141.354150:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354154:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764141.354156:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354160:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354166:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764141.354171:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764141.354177:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764141.354186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764141.354193:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067778176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764141.354205:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354208:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764141.354210:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354213:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354218:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764141.354222:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764141.354227:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764141.354228:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020dac380 x1876702067777664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764141.354241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067778304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764141.354250:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067777664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764141.354251:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764141.354262:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764141.354269:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764141.354282:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1609 00000100:00080000:0.0:1789764141.354287:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764141.354288:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764141.354299:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764141.354300:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:0.0:1789764141.354309:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764141.354317:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067778432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764141.354326:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067777664:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764141.354338:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764141.354366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354379:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020daf480 x1876702067777792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354391:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067777792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354399:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764141.354430:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764141.354452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354463:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020dae680 x1876702067777920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067777920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354486:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067777920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764141.354496:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764141.354515:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020dad180 x1876702067778048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067778048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354543:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354547:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764141.354555:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764141.354575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020dae300 x1876702067778176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067778176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354611:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764141.354617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067778304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764141.354634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020dadf80 x1876702067778304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354651:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067778304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067778304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764141.354665:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067778432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764141.354689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764141.354700:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764141/real 1789764141] req@ffff936020daf800 x1876702067778432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764141.354711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067778432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764166 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764141.354730:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764141.354733:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764141.354736:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067778432:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764142.121983:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764143.145976:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764144.170113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764145.193989:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764146.217992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764146.473909:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.473920:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764146.473925:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764146.473936:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.473949:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.473959:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764146.473998:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067778560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764146.474021:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474025:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764146.474027:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764146.474031:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.474039:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.474044:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764146.474055:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764146.474061:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067778560:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764146.474071:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067778688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764146.474084:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474087:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764146.474089:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764146.474094:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.474098:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.474103:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764146.474108:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764146.474120:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067778816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764146.474129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764146.474134:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474138:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764146.474140:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764146.474146:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.474151:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.474155:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764146.474160:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764146.474172:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef6680 x1876702067778560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764146.474176:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067778944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764146.474200:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474203:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764146.474206:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764146.474208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067778560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764146.474212:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.474218:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.474224:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764146.474225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789764146.474230:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789764146.474231:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067778560:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764146.474251:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067779072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764146.474261:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778688:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764146.474264:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474269:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764146.474271:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764146.474276:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00080000:1.0:1789764146.474280:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764146.474285:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764146.474290:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764146.474292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764146.474308:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067779200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764146.474309:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef5f80 x1876702067778688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474339:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067778688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764146.474340:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764146.474349:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764146.474354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764146.474354:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474360:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764146.474364:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1614 00000100:00100000:2.0:1789764146.474366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778688:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764146.474370:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764146.474373:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778816:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764146.474378:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764146.474384:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764146.474393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764146.474400:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067779328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764146.474404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef6a00 x1876702067778816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067778816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764146.474438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764146.474451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764146.474469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764146.474476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef5500 x1876702067778944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474484:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067778944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764146.474491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474502:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067778944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764146.474521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764146.474539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764146.474551:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef7800 x1876702067779072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067779072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764146.474566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764146.474573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067779200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764146.474600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764146.474612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef7480 x1876702067779200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067779200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764146.474641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067779200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764146.474653:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067779328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764146.474672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764146.474691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764146/real 1789764146] req@ffff936010ef7b80 x1876702067779328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764146.474699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067779328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764146.474704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764146.474706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764146.474707:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067779328:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764147.242107:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764148.265979:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764149.290431:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764150.314192:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764151.337956:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764151.594082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764151.594103:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594113:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594125:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594132:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764151.594183:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067779456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764151.594217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764151.594223:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594227:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594234:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594245:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764151.594258:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067779584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764151.594275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764151.594280:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594285:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00100000:2.0:1789764151.594287:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067779456:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764151.594290:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594298:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764151.594317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067779712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764151.594326:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594330:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764151.594332:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594335:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594340:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594344:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594349:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764151.594361:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067779840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764151.594370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764151.594375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764151.594381:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594386:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594391:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594395:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594400:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764151.594410:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067779968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764151.594418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764151.594422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764151.594426:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764151.594427:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef6300 x1876702067779456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764151.594434:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594441:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594447:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594453:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764151.594457:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067779456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764151.594469:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067780096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764151.594470:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594480:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764151.594486:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764151.594488:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067779456:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764151.594493:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764151.594496:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764151.594501:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1619 00000100:00080000:1.0:1789764151.594505:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764151.594509:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764151.594514:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764151.594522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764151.594527:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067780224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764151.594552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef6d80 x1876702067779584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067779584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594595:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764151.594601:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764151.594616:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594625:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef4e00 x1876702067779712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067779712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594662:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764151.594667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764151.594687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef7100 x1876702067779840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594710:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067779840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594717:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764151.594725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764151.594738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef4380 x1876702067779968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067779968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594759:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067779968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764151.594762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764151.594772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594778:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef4000 x1876702067780096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067780096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594789:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764151.594825:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764151.594837:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764151.594844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764151/real 1789764151] req@ffff936010ef5c00 x1876702067780224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764151.594850:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067780224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764151.594855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764151.594857:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764151.594858:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780224:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764152.362047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764153.386130:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764154.410046:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764155.433998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764156.458034:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764156.713917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.713928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764156.713932:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764156.713942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.713952:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.713960:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764156.713991:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067780352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764156.714019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764156.714025:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764156.714028:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714032:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714036:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714045:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764156.714058:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067780480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764156.714069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764156.714075:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764156.714078:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067780352:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764156.714082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714089:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714092:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714097:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764156.714110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067780608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764156.714121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764156.714126:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764156.714131:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714137:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714140:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714144:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764156.714150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764156.714155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067780736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764156.714166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764156.714172:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764156.714175:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714179:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714184:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714189:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764156.714200:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067780864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764156.714202:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef4a80 x1876702067780352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764156.714235:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764156.714240:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764156.714245:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067780352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764156.714246:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714252:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714258:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789764156.714269:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789764156.714276:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067780992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764156.714277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067780352:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764156.714294:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764156.714298:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764156.714300:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764156.714304:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1624 00000100:00080000:1.0:1789764156.714307:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764156.714311:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764156.714315:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764156.714317:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764156.714340:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067781120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764156.714350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714381:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef5180 x1876702067780480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714394:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067780480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714403:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714406:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764156.714421:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764156.714445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef4700 x1876702067780608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067780608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714485:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764156.714508:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764156.714531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714545:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef5880 x1876702067780736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714556:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067780736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764156.714578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764156.714597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef5c00 x1876702067780864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067780864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067780864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764156.714657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764156.714686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef4000 x1876702067780992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067780992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714708:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067780992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764156.714724:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067781120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764156.714741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764156.714751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764156/real 1789764156] req@ffff936010ef4380 x1876702067781120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764156.714762:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067781120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764156.714770:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764156.714773:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764156.714776:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067781120:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764157.482010:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764158.506230:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764159.530026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764160.553996:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764161.578095:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764161.834549:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.834562:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764161.834577:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.834586:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.834596:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.834603:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764161.834761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067781248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764161.834847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067781248:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764161.834924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764161.835084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.835089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764161.835091:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.835096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.835100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.835105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.835113:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764161.835127:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067781376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764161.835184:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061de680 x1876702067781248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.835201:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067781248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.835216:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.835219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.835223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067781248:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764161.835254:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764161.835279:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764161.835292:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dc000 x1876702067781376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.835304:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067781376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.835312:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.835315:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.835318:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781376:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764161.835451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.835464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764161.835467:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.835471:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.835476:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.835479:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.835485:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764161.835498:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067781504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764161.835652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764161.835676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764161.835776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.835781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764161.835782:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.835787:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.835791:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.835795:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.835816:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764161.835829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067781632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764161.835905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dd180 x1876702067781504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.835929:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067781504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.835945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.835949:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.835952:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764161.835967:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764161.835990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764161.836002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dce00 x1876702067781632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.836013:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067781632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.836021:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.836024:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.836027:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781632:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789764161.836296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.836300:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764161.836302:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.836306:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.836310:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.836314:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.836320:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764161.836332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067781760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764161.836350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.836353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764161.836355:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.836358:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.836361:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.836364:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.836368:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764161.836376:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067781888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764161.836383:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764161.836386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764161.836388:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764161.836391:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1629 00000100:00080000:2.0:1789764161.836393:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764161.836397:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764161.836401:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764161.836407:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067782016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764161.836465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764161.836486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764161.836496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dce00 x1876702067781760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.836515:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067781760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.836523:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.836525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.836528:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067781760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764161.836534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067781888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764161.836551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764161.836562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dd180 x1876702067781888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.836584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067781888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.836591:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.836594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.836597:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067781888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764161.836602:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764161.836823:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764161.836836:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764161/real 1789764161] req@ffff9360061dc000 x1876702067782016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764161.836847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067782016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764161.836855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764161.836858:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764161.836861:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782016:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764162.603548:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764163.627124:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764164.650127:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764165.674212:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764166.698030:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764166.953913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.953925:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764166.953930:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.953942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.953956:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.953964:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764166.954011:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067782144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764166.954094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067782144:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764166.954163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764166.954196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.954199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764166.954202:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.954206:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.954209:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.954214:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.954225:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764166.954240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067782272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764166.955388:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061de680 x1876702067782144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.955409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067782144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.955427:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.955431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.955435:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067782144:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764166.955461:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764166.955494:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.955507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061dfb80 x1876702067782272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.955519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067782272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.955539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.955543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.955546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782272:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764166.956420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.956425:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764166.956428:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.956433:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.956438:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.956444:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.956450:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764166.956465:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067782400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764166.956478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.956482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764166.956484:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.956488:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.956491:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.956495:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.956500:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764166.956511:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067782528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764166.956521:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.956524:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764166.956526:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.956544:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.956548:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.956552:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.956556:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764166.956567:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067782656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764166.956581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.956585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764166.956587:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.956590:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.956594:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.956598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.956603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764166.956623:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067782784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764166.956632:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764166.956636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764166.956638:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764166.956641:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1634 00000100:00080000:2.0:1789764166.956645:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764166.956649:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764166.956653:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764166.956663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067782912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764166.956741:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764166.956767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.956781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061df800 x1876702067782400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.956794:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067782400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.956819:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.956822:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.956826:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764166.956833:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764166.956855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.956867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061dca80 x1876702067782528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.956879:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067782528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.956887:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.956891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.956894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764166.956900:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764166.956919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.956929:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061dd500 x1876702067782656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.956941:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067782656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.956948:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.956951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.956954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067782656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764166.956960:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764166.956978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.956990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061df480 x1876702067782784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.957002:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067782784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.957010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.957013:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.957016:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764166.957021:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764166.957040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764166.957051:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764166/real 1789764166] req@ffff9360061df100 x1876702067782912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764166.957062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067782912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764166.957070:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764166.957073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764166.957075:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067782912:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764167.722087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764168.746252:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764169.770405:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764170.794031:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764171.818116:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764172.074877:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.074889:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764172.074894:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.074903:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.074914:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.074921:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764172.074967:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067783040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.074998:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075001:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764172.075003:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075006:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075010:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075013:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075022:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764172.075033:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067783168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.075042:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075046:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764172.075047:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075051:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075054:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075057:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075061:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764172.075070:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067783296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.075079:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075082:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764172.075083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075086:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075090:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075093:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075097:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764172.075108:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067783424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.075117:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075120:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764172.075122:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075125:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075128:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075131:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075135:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764172.075144:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067783552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.075151:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075155:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764172.075156:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075159:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075162:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075165:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075170:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764172.075179:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067783680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764172.075193:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764172.075196:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764172.075197:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764172.075200:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1639 00000100:00080000:0.0:1789764172.075203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764172.075206:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764172.075210:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764172.075220:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067783808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764172.075242:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067783040:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764172.075299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075347:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020daea00 x1876702067783040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067783040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075381:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075390:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067783040:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764172.075397:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764172.075418:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075430:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020dace00 x1876702067783168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067783168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075457:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764172.075462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764172.075477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020dadc00 x1876702067783296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067783296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764172.075531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764172.075552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020daf800 x1876702067783424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075573:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067783424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075581:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764172.075593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764172.075609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020dadf80 x1876702067783552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075629:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067783552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075642:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067783552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764172.075646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067783680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764172.075663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020dae300 x1876702067783680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067783680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075692:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067783680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764172.075700:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067783808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764172.075715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764172.075724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764172/real 1789764172] req@ffff936020dad180 x1876702067783808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764172.075735:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067783808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764172.075741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764172.075744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764172.075746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067783808:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764172.841982:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764173.866220:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764174.891005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764175.914202:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764176.938097:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764177.194216:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194228:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764177.194233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194243:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194256:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194263:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764177.194308:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067783936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194337:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194340:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764177.194342:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194345:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194347:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194349:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194358:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764177.194366:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067784064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194372:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194374:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764177.194375:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194378:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194379:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194381:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194383:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764177.194389:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067784192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194395:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194397:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764177.194398:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194401:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194403:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194405:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194407:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764177.194412:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067784320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194416:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194418:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764177.194419:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194420:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194422:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194424:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194426:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764177.194431:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067784448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194436:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194438:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764177.194438:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194441:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194443:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194445:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194447:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764177.194466:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067784576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764177.194471:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764177.194473:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764177.194474:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764177.194476:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1645 00000100:00080000:1.0:1789764177.194478:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764177.194480:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764177.194484:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764177.194489:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067784704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764177.194570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067783936:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764177.194638:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.194695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef7100 x1876702067783936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.194716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067783936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.194727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.194731:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.194735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067783936:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764177.194742:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764177.194769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.194890:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef4e00 x1876702067784064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.194906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067784064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.194915:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.194918:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.194921:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764177.194927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764177.194948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.194960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef6d80 x1876702067784192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.194970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067784192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.194982:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.194985:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.194988:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764177.194993:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764177.195014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.195025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef6300 x1876702067784320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.195140:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067784320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.195149:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.195153:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.195156:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764177.195162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764177.195186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.195198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef7b80 x1876702067784448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.195210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067784448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.195218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.195221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.195225:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764177.195231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067784576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764177.195249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.195260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef7480 x1876702067784576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.195271:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067784576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.195278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.195281:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.195285:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067784576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764177.195290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067784704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764177.195307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764177.195318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764177/real 1789764177] req@ffff936010ef7800 x1876702067784704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764177.195330:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067784704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764177.195337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764177.195341:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764177.195344:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067784704:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764177.963767:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764178.986181:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764180.010085:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764181.035052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764182.058001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764182.313987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764182.314061:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764182.314072:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314087:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314097:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764182.314144:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067784832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764182.314179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764182.314185:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764182.314189:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314193:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314196:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314208:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764182.314221:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067784960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764182.314230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067784832:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764182.314234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764182.314242:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764182.314247:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314250:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314255:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314260:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764182.314272:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067785088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764182.314282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764182.314288:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764182.314291:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314446:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314452:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314458:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764182.314464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764182.314472:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067785216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764182.314483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764182.314488:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764182.314492:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314496:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314499:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314504:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764182.314515:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067785344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764182.314523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764182.314527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764182.314530:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764182.314531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa4000 x1876702067784832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764182.314538:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314545:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314552:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314556:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764182.314562:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067784832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764182.314571:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067785472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764182.314578:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764182.314584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764182.314587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764182.314590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764182.314594:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764182.314595:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067784832:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764182.314603:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1650 00000100:00080000:3.0:1789764182.314608:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764182.314612:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764182.314617:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764182.314632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764182.314642:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067785600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764182.314660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.314674:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa7100 x1876702067784960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.314687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067784960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.314695:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.314699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.314702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067784960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764182.314707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764182.314724:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.314734:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa4700 x1876702067785088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.314744:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067785088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.314757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.314760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.314763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764182.314768:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764182.314787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.314824:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa5880 x1876702067785216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.314836:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067785216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.314844:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.314847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.314850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764182.314856:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764182.314877:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.314888:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa4a80 x1876702067785344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.314901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067785344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.314908:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.314911:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.314914:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764182.314920:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067785472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764182.314939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.314950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa4380 x1876702067785472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.314961:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067785472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.314969:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.314972:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.314975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067785472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764182.314994:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067785600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764182.315013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764182.315023:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764182/real 1789764182] req@ffff936010aa5180 x1876702067785600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764182.315033:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067785600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764207 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764182.315041:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764182.315044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764182.315046:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067785600:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764183.082694:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764184.106678:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764185.130536:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764186.154277:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764187.177980:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764187.434038:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434050:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764187.434054:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434064:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434074:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434081:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764187.434119:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067785728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434153:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434157:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764187.434159:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434163:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434166:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434170:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434180:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764187.434192:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067785856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434202:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434204:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764187.434214:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434219:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434223:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434226:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764187.434241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067785984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434249:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434252:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764187.434254:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434257:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434260:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434264:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434268:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764187.434277:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067786112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764187.434289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434293:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434296:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434299:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434303:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764187.434311:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067786240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434320:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434323:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764187.434324:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434328:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434330:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434334:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434338:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764187.434347:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067786368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764187.434356:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764187.434358:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764187.434372:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764187.434377:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1655 00000100:00080000:0.0:1789764187.434391:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764187.434395:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764187.434399:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764187.434410:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067786496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764187.434681:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067785728:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764187.434874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.434930:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020dae680 x1876702067785728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.434948:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067785728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.434959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.434964:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.434968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067785728:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764187.434987:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764187.435013:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020daf480 x1876702067785856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067785856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435044:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435047:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435050:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764187.435056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764187.435075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435085:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020dac380 x1876702067785984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435095:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067785984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435111:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435114:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067785984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764187.435120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764187.435138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435149:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020daf100 x1876702067786112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435160:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067786112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435167:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435173:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764187.435178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764187.435195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435207:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020dac700 x1876702067786240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435218:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067786240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435228:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435232:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764187.435238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067786368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764187.435258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435269:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020dafb80 x1876702067786368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067786368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067786368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764187.435320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067786496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764187.435341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764187.435352:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764187/real 1789764187] req@ffff936020dac000 x1876702067786496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764187.435363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067786496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764187.435370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764187.435374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764187.435376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067786496:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764188.203889:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764189.226005:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764190.249969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764191.274474:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764192.298023:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764192.554176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764192.554190:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554202:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554214:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554222:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764192.554260:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067786624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764192.554293:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554297:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00100000:2.0:1789764192.554331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067786624:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764192.554381:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554387:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554396:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764192.554409:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067786752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764192.554423:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554426:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554429:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554431:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554434:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764192.554442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764192.554451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067786880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764192.554467:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554472:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554475:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554480:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554488:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764192.554497:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa6300 x1876702067786624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764192.554505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067787008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554529:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764192.554531:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764192.554537:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067786624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764192.554544:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554551:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554554:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554559:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764192.554570:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067787136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554588:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764192.554592:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554596:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554599:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554603:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554607:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764192.554619:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067787264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764192.554627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764192.554630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764192.554632:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764192.554635:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1660 00000100:00080000:3.0:1789764192.554638:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764192.554641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764192.554644:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764192.554651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067787392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764192.554686:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.554693:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.554701:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067786624:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764192.554736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764192.554767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.554785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa7800 x1876702067786752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555092:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067786752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555101:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555108:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764192.555115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764192.555141:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.555153:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa7b80 x1876702067786880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555166:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067786880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555181:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555185:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555189:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067786880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764192.555199:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764192.555223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.555241:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa5180 x1876702067787008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555252:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067787008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555261:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555265:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555272:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764192.555282:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764192.555304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.555314:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa4380 x1876702067787136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555324:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067787136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555331:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555337:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764192.555361:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067787264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764192.555380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.555390:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa4a80 x1876702067787264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555400:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067787264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555408:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067787264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764192.555419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067787392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764192.555433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764192.555441:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764192/real 1789764192] req@ffff936010aa5880 x1876702067787392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764192.555451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067787392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764192.555458:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764192.555461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764192.555463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067787392:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764193.322371:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764194.346506:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764195.372062:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764196.394028:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764197.418116:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764197.674180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674188:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764197.674193:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674202:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674213:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674220:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764197.674260:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067787520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764197.674325:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067787520:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764197.674337:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764197.674344:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674348:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674352:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674357:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674367:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764197.674379:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067787648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764197.674388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674391:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764197.674393:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674396:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674399:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674402:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674406:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764197.674415:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067787776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764197.674423:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674427:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764197.674428:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674432:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674435:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674438:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674443:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764197.674451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067787904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764197.674456:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787648:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764197.674460:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674462:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764197.674464:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674467:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674470:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674477:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764197.674483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764197.674485:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067788032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764197.674494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764197.674498:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674502:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00100000:2.0:1789764197.674505:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787904:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764197.674505:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674509:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674513:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764197.674539:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764197.674602:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067788160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764197.674655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067788160:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764197.674698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764197.674702:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764197.674703:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764197.674707:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1665 00000100:00080000:3.0:1789764197.674712:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764197.674715:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764197.674720:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764197.674731:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067788288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764197.674759:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067788288:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764198.441998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764199.467345:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764200.490389:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789764200.746176:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:0.0:1789764200.746226:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789764200.746246:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764200.746690:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa4700 x1876702067787520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.746721:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067787520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.746734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.746739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.746744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067787520:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764200.746876:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764200.748699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa7100 x1876702067787648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.748734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067787648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.748746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.748751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.748755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764200.749417:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764200.749753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa4000 x1876702067787776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.749776:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067787776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.749787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.749792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.749796:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764200.750404:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764200.751392:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764200.751408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa5c00 x1876702067787904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789764200.751413:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764200.751441:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789764200.751441:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067787904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.751457:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.751464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.751469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067787904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764200.751477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa6a00 x1876702067788032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.751489:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067788032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.751498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.751502:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.751505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764200.751528:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa6680 x1876702067788288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.751542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067788288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.751550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.751554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.751557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067788288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764200.751564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764197/real 1789764200] req@ffff936010aa5f80 x1876702067788160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764200.751576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067788160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764200.751584:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764200.751588:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764200.751591:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067788160:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764201.515047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764202.539711:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764202.794011:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.794022:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764202.794026:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.794038:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.794051:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.794060:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764202.794103:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067788416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.794137:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.794141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764202.794143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.794147:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.794151:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.794155:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764202.794725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067788416:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764202.794824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.794871:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020daed80 x1876702067788416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.794891:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067788416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.794903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.794908:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.794911:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067788416:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789764202.795315:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764202.795336:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067788544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.795348:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.795352:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764202.795354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.795360:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.795365:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.795370:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764202.795376:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764202.795389:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067788672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.795399:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.795403:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764202.795404:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.795408:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.795412:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.795416:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764202.795551:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764202.795564:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067788800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.795573:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.795577:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764202.795579:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.795582:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.795586:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.795590:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764202.795595:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764202.795603:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067788928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.795612:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.795615:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764202.795617:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.795619:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.795622:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.795625:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764202.795628:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764202.795635:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067789056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764202.795643:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764202.795645:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764202.795646:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764202.795649:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1670 00000100:00080000:0.0:1789764202.795652:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764202.795655:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764202.795659:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764202.795666:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067789184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764202.796224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764202.796289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.796316:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020dad500 x1876702067788544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.796335:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067788544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.796345:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.796349:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.796353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764202.796361:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764202.796384:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.796397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020dad880 x1876702067788672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.796412:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067788672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.796421:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.796424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.796427:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764202.796433:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764202.796455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.796466:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020daed80 x1876702067788800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.796478:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067788800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.796486:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.796489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.796492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764202.796498:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764202.796518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.796528:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020dac000 x1876702067788928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.796538:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067788928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.796545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.796549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.796551:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067788928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764202.796557:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764202.796577:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.796588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020dafb80 x1876702067789056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.796607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067789056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.796615:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.796617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.796620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764202.796626:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764202.798435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764202.798478:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764202/real 1789764202] req@ffff936020dac700 x1876702067789184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764202.798500:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067789184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764202.798511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764202.798515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764202.798519:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789184:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764203.562004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764204.586132:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764205.610222:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764206.634114:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764207.658441:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764207.913984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.913992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764207.913996:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914005:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914016:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914022:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764207.914062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067789312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764207.914095:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914100:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914111:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764207.914119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067789440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764207.914129:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914132:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914133:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914135:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914137:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764207.914143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067789568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764207.914152:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914154:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914156:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914158:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914161:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764207.914168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067789696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764207.914176:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914178:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914179:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914183:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764207.914188:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067789824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764207.914196:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914198:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914199:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914201:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914203:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764207.914208:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067789952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764207.914213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764207.914215:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764207.914217:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764207.914219:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1675 00000100:00080000:3.0:1789764207.914221:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764207.914223:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764207.914225:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764207.914240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067790080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764207.915397:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067789312:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764207.915475:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa5500 x1876702067789312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915540:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067789312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915560:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915564:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067789312:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764207.915572:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764207.915603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa4e00 x1876702067789440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067789440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915643:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764207.915654:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764207.915672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa7480 x1876702067789568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067789568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915710:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764207.915720:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764207.915741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa5f80 x1876702067789696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915779:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067789696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915788:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764207.915836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764207.915859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa6680 x1876702067789824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067789824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067789824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764207.915905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764207.915926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.915954:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa6a00 x1876702067789952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.915965:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067789952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.915972:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.915975:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.915978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067789952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764207.915985:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764207.916005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764207.916016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764207/real 1789764207] req@ffff936010aa5c00 x1876702067790080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764207.916026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067790080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764207.916033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764207.916036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764207.916039:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790080:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764208.682158:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764209.706449:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764210.732071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764211.753969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764212.777997:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764213.033910:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.033923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764213.033927:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.033937:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.033948:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.033957:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764213.034002:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067790208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764213.034028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764213.034032:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034035:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034037:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034040:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034048:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764213.034059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067790336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764213.034066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764213.034069:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034072:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764213.034087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067790464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764213.034089:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067790208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764213.034098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764213.034102:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034104:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034106:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034108:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034110:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764213.034117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067790592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764213.034122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764213.034124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034132:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034135:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764213.034141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067790720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764213.034146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764213.034149:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034155:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034158:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034161:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764213.034169:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067790848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764213.034177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764213.034177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764213.034181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764213.034183:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764213.034187:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1680 00000100:00080000:3.0:1789764213.034189:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764213.034201:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764213.034204:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764213.034212:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067790976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764213.034262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa4000 x1876702067790208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034286:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067790208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034304:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034308:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067790208:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764213.034320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764213.034345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034359:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa7100 x1876702067790336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067790336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034381:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034385:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034389:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764213.034426:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764213.034447:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034459:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa4700 x1876702067790464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034471:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067790464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034479:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034483:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034486:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764213.034491:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764213.034511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa5880 x1876702067790592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067790592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034545:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764213.034554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764213.034573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034584:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa4a80 x1876702067790720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034594:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067790720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034601:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067790720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764213.034612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764213.034629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa4380 x1876702067790848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067790848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764213.034669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764213.034687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764213.034697:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764213/real 1789764213] req@ffff936010aa5180 x1876702067790976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764213.034709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067790976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764213.034717:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764213.034719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764213.034722:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067790976:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764213.801981:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764214.826253:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764215.850048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764216.873979:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764217.897973:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764218.153976:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.153987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764218.153992:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154004:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154018:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154027:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764218.154075:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067791104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764218.154114:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154119:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154126:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764218.154138:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764218.154150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067791232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764218.154184:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154188:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154192:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154196:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764218.154200:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764218.154210:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067791360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764218.154225:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154230:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154233:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154237:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764218.154243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067791104:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789764218.154244:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764218.154260:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067791488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764218.154277:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154281:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154284:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154288:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764218.154293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764218.154304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067791616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154317:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764218.154320:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764218.154321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764218.154327:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154333:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154337:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764218.154343:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764218.154355:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067791744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764218.154422:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764218.154426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764218.154428:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764218.154432:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1685 00000100:00080000:3.0:1789764218.154435:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764218.154439:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764218.154443:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764218.154454:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067791872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764218.154655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa7b80 x1876702067791104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.154688:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067791104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.154707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.154711:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.154714:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067791104:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764218.154722:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764218.154750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.154762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa7800 x1876702067791232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.154772:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067791232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.154779:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.154782:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.154785:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764218.154790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764218.154841:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.154850:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa6300 x1876702067791360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.154860:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067791360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.154867:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.154870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.154873:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764218.154903:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764218.154922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.154932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa5180 x1876702067791488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.154943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067791488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.154951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.154954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.154957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764218.154963:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764218.154979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.154989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa4380 x1876702067791616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.154999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067791616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.155012:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.155014:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.155017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067791616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764218.155022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067791744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764218.155039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.155048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa4a80 x1876702067791744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.155058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067791744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.155065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.155068:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.155070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067791744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764218.155076:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067791872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764218.155091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764218.155100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764218/real 1789764218] req@ffff936010aa5880 x1876702067791872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764218.155110:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067791872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764218.155116:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764218.155119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764218.155122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067791872:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764218.922044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764219.945997:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764220.970016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764221.994065:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764223.018149:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764223.273925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.273939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764223.273943:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.273953:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.273963:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.273970:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764223.274012:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067792000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764223.274097:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067792000:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764223.274185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764223.274217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.274221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764223.274223:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.274227:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.274231:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.274235:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.274244:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764223.274258:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067792128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764223.274584:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa4700 x1876702067792000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.274604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067792000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.274631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.274634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.274637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067792000:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764223.274666:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764223.274691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764223.274705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa7100 x1876702067792128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.274717:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067792128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.274725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.274728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.274731:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792128:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764223.275249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.275254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764223.275256:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.275261:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.275266:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.275270:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.275276:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764223.275290:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067792256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764223.275337:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764223.275363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764223.275383:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.275386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764223.275388:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.275392:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.275396:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.275400:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.275405:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764223.275418:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067792384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764223.275462:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa4000 x1876702067792256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.275475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067792256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.275483:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.275487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.275490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764223.275503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764223.275523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764223.275534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa5c00 x1876702067792384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.275544:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067792384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.275552:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.275555:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.275558:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792384:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764223.275602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.275607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764223.275608:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.275622:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.275847:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.275853:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.275859:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764223.275944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067792512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764223.275955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.275958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764223.275960:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.275964:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.275968:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.275972:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.275977:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764223.275986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067792640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764223.276183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764223.276186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764223.276187:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764223.276192:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1691 00000100:00080000:3.0:1789764223.276196:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764223.276200:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764223.276205:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764223.276217:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067792768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764223.276259:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764223.276285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764223.276298:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa6a00 x1876702067792512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.276310:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067792512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.276318:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.276321:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.276324:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067792512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764223.276341:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067792640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764223.276362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764223.276374:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa6680 x1876702067792640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.276386:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067792640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.276393:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.276396:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.276399:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067792640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764223.276405:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067792768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764223.276424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764223.276436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764223/real 1789764223] req@ffff936010aa5f80 x1876702067792768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764223.276448:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067792768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764248 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764223.276455:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764223.276458:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764223.276462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067792768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764224.041973:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764225.066602:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764226.090094:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764227.114121:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764228.139604:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764228.394316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394325:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764228.394331:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394345:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394368:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394378:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764228.394417:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067792896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394447:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764228.394457:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394461:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394466:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764228.394481:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764228.394492:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067792896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764228.394496:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067793024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764228.394516:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394521:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394525:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394530:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764228.394535:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764228.394548:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067793152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394561:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764228.394563:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394567:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394571:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789764228.394572:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764228.394580:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764228.394586:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764228.394599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067793280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394612:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764228.394613:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394617:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394621:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394624:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764228.394629:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764228.394631:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020daf100 x1876702067792896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764228.394643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067793408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789764228.394659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067792896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764228.394661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764228.394665:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394676:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00080000:0.0:1789764228.394682:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394690:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:3.0:1789764228.394692:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789764228.394696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789764228.394699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.394705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067792896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764228.394712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067793536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764228.394722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764228.394725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764228.394727:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764228.394733:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1696 00000100:00100000:3.0:1789764228.394733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793024:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764228.394741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764228.394746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764228.394751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764228.394764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067793664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764228.394778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.394824:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020dac380 x1876702067793024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.394839:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067793024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.394848:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.394852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.394856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764228.394863:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764228.394883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.394892:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020daf480 x1876702067793152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.394904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067793152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.394985:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.394989:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.394994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764228.395001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764228.395023:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.395034:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020dae680 x1876702067793280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.395045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067793280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.395182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.395186:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.395190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764228.395196:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764228.395218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.395230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020dad180 x1876702067793408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.395245:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067793408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.395291:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.395295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.395299:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764228.395306:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067793536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764228.395338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.395349:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020dae300 x1876702067793536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.395361:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067793536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.395369:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.395372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.395375:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067793536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764228.395393:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067793664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764228.395410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764228.395419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764228/real 1789764228] req@ffff936020dadf80 x1876702067793664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764228.395430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067793664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764253 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764228.395438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764228.395440:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764228.395443:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067793664:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764229.162602:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764230.186252:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764231.210444:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764232.234165:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764233.260266:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764233.513938:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.513948:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764233.513953:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.513965:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.513983:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.513992:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764233.514065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067793792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764233.514146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067793792:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764233.514231:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764233.514266:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514271:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764233.514273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514278:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514282:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514287:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514299:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764233.514313:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067793920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764233.514526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061dd880 x1876702067793792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.514553:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067793792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.514565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.514570:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.514574:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067793792:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764233.514604:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764233.514630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.514644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061ddc00 x1876702067793920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.514656:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067793920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.514665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.514668:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.514671:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067793920:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764233.514766:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514772:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764233.514774:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514778:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514783:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514788:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514794:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764233.514827:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067794048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764233.514838:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514842:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764233.514843:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514847:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514851:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514855:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514860:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764233.514869:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067794176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764233.514878:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514881:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764233.514883:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514886:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514890:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514893:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514898:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764233.514907:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067794304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764233.514915:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514918:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764233.514920:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514924:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514927:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514931:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514936:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764233.514945:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067794432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764233.514958:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764233.514961:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764233.514963:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764233.514967:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1701 00000100:00080000:2.0:1789764233.514970:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764233.514974:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764233.514978:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764233.514988:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067794560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764233.515095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764233.515121:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.515134:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061de300 x1876702067794048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.515146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067794048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.515161:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.515165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.515168:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764233.515174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764233.515197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.515209:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061ddf80 x1876702067794176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.515220:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067794176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.515229:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.515233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.515236:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764233.515242:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764233.515262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.515273:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061dc700 x1876702067794304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.515284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067794304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.515292:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.515295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.515299:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764233.515304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067794432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764233.515325:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.515336:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061dc380 x1876702067794432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.515348:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067794432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.515355:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.515358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.515361:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067794432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764233.515367:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067794560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764233.515386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764233.515397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764233/real 1789764233] req@ffff9360061ddc00 x1876702067794560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764233.515409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067794560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764233.515417:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764233.515419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764233.515423:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067794560:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764234.282289:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764235.306514:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764236.330064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764237.354044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764238.378431:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764238.634388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764238.634405:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634417:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634432:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634439:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764238.634485:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067794688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634523:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764238.634525:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634529:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634532:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634536:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634547:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764238.634557:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067794816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764238.634570:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634577:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634580:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634584:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764238.634593:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067794944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764238.634606:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634610:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634622:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634627:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634631:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764238.634642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067794688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764238.634653:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067795072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764238.634671:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634675:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634678:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634682:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764238.634701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067795200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764238.634715:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634718:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634721:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634728:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:3.0:1789764238.634728:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764238.634748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067795328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764238.634762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764238.634765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764238.634767:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764238.634771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1706 00000100:00080000:0.0:1789764238.634775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764238.634779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764238.634784:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764238.634792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067795456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764238.635092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020daf800 x1876702067794688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635116:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067794688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635127:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635135:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067794688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764238.635143:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764238.635170:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635184:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020dadc00 x1876702067794816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635196:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067794816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635204:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635208:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635212:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764238.635218:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764238.635239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635252:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020dace00 x1876702067794944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635263:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067794944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635270:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635273:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635276:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067794944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764238.635306:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764238.635328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635340:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020daea00 x1876702067795072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067795072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764238.635371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764238.635389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635400:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020dadf80 x1876702067795200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635410:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067795200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635418:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635421:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764238.635429:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067795328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764238.635445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020dae300 x1876702067795328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635463:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067795328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067795328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764238.635483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067795456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764238.635500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764238.635510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764238/real 1789764238] req@ffff936020dad180 x1876702067795456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764238.635521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067795456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764238.635528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764238.635531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764238.635534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067795456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764239.402021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764240.426060:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764241.453489:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764242.474018:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764243.498052:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764243.754065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764243.754082:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754094:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754165:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754174:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764243.754230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067795584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764243.754271:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754275:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754279:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754283:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754294:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764243.754307:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067795712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764243.754322:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754326:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754330:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754333:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754338:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764243.754348:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067795840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754357:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764243.754362:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754366:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754369:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754372:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754376:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764243.754386:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067795968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754398:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764243.754400:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754403:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754406:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754410:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754415:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764243.754425:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067796096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754434:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754437:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764243.754439:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754442:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754445:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754449:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754453:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764243.754460:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067796224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764243.754469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764243.754472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764243.754473:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764243.754477:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1711 00000100:00080000:1.0:1789764243.754479:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764243.754482:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764243.754486:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764243.754495:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067796352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764243.756902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067795584:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764243.756999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef5500 x1876702067795584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757098:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067795584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757112:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757116:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757120:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067795584:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764243.757128:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764243.757151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef6a00 x1876702067795712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067795712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764243.757216:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764243.757235:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757244:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef5f80 x1876702067795840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757256:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067795840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757271:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764243.757277:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764243.757295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757304:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef6680 x1876702067795968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757315:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067795968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757339:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067795968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764243.757345:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764243.757362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef7800 x1876702067796096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067796096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757412:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757416:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757419:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764243.757425:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067796224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764243.757443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef7480 x1876702067796224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067796224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067796224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764243.757484:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067796352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764243.757499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764243.757507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764243/real 1789764243] req@ffff936010ef7b80 x1876702067796352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764243.757518:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067796352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764243.757532:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764243.757536:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764243.757538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067796352:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764244.522128:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764245.545995:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764246.570239:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764247.594354:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764248.618901:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764248.874023:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764248.874040:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874052:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874065:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764248.874074:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764248.874119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067796480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764248.874152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764248.874158:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874162:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874166:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764248.874169:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874181:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764248.874194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067796608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764248.874211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874215:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764248.874217:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874222:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874227:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764248.874236:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067796480:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764248.874236:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874244:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764248.874259:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067796736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764248.874272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764248.874279:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874283:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874300:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764248.874306:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874311:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764248.874319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764248.874333:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067796864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764248.874345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764248.874350:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874354:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874359:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764248.874364:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874369:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764248.874380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa7480 x1876702067796480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764248.874385:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067796992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764248.874425:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874432:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789764248.874433:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067796480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764248.874438:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874449:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874460:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764248.874463:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764248.874470:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874478:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789764248.874481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.874491:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067796480:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764248.874499:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067797120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764248.874507:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796608:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764248.874520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764248.874525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764248.874528:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764248.874534:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1716 00000100:00080000:3.0:1789764248.874539:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764248.874548:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:2.0:1789764248.874550:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789764248.874560:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764248.874574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa4e00 x1876702067796608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764248.874580:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067797248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764248.874617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067796608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.874632:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.874636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.874640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764248.874671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764248.874693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764248.874703:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa5500 x1876702067796736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764248.874715:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067796736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.874722:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.874726:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.874729:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764248.874735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764248.874755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764248.874765:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa5f80 x1876702067796864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764248.874791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067796864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.874845:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.874850:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.874853:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764248.874860:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764248.874882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764248.874894:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa6680 x1876702067796992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764248.874906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067796992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.874914:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.874917:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.874920:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067796992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764248.874927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067797120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764248.874960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764248.874972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa6a00 x1876702067797120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764248.874992:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067797120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.875000:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.875004:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.875008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067797120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764248.875014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067797248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764248.875034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764248.875045:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764248/real 1789764248] req@ffff936010aa5c00 x1876702067797248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764248.875057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067797248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764248.875064:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764248.875067:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764248.875070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067797248:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764249.642048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764250.666838:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764251.690040:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764252.714958:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764253.738014:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764253.994047:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994061:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764253.994066:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994077:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994088:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994096:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764253.994134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067797376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764253.994163:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994167:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764253.994169:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994172:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994176:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994180:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994189:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764253.994206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067797504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764253.994206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067797376:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764253.994221:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764253.994227:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994230:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994234:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994237:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764253.994254:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067797632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764253.994264:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994267:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764253.994268:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994271:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1721 00000800:00000100:3.0:1789764253.994277:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764253.994278:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994284:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994289:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764253.994300:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067797760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764253.994308:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994312:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764253.994313:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994316:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994319:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994323:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994328:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764253.994335:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061dd880 x1876702067797376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764253.994340:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067797888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764253.994359:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994362:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764253.994365:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789764253.994365:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067797376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764253.994374:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994381:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764253.994384:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764253.994386:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994391:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789764253.994399:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764253.994401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067798016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764253.994406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067797376:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764253.994419:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764253.994422:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764253.994424:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764253.994428:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1721 00000100:00080000:2.0:1789764253.994431:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764253.994434:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764253.994439:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764253.994441:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764253.994451:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067798144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764253.994475:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061df100 x1876702067797504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994495:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067797504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994500:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994502:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764253.994508:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764253.994521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061df480 x1876702067797632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067797632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764253.994565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764253.994585:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994596:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061dd500 x1876702067797760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067797760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994629:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764253.994637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764253.994657:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061dca80 x1876702067797888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067797888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994686:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994689:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994692:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067797888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764253.994696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764253.994712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994723:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061df800 x1876702067798016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994733:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067798016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994743:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764253.994765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764253.994784:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764253.994794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764253/real 1789764253] req@ffff9360061dfb80 x1876702067798144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764253.994830:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067798144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764253.994843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764253.994857:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764253.994861:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798144:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764254.762110:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764255.786599:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764256.810729:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764257.834139:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764258.858087:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764259.114056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764259.114072:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114085:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:3.0:1789764259.114099:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114107:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764259.114155:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067798272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764259.114180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764259.114185:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114188:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:3.0:1789764259.114191:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114194:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764259.114203:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764259.114216:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067798400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764259.114220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067798272:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764259.114240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764259.114246:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114251:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:3.0:1789764259.114255:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114258:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764259.114262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764259.114312:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764259.114341:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067798528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764259.114355:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114359:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764259.114361:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114365:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:3.0:1789764259.114370:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114375:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764259.114382:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764259.114384:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa4000 x1876702067798272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764259.114397:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067798656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764259.114408:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067798272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764259.114408:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114417:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764259.114422:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114430:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:2.0:1789764259.114431:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764259.114435:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764259.114440:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764259.114445:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067798272:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789764259.114452:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764259.114464:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067798784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764259.114470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764259.114474:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764259.114480:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00100000:2.0:1789764259.114481:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798400:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764259.114483:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114489:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764259.114492:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764259.114499:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067798912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764259.114505:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764259.114508:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764259.114510:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764259.114514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764259.114516:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1726 00000100:00080000:3.0:1789764259.114520:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764259.114523:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764259.114526:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764259.114535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa7100 x1876702067798400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764259.114535:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067799040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764259.114569:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067798400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114583:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114587:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764259.114594:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764259.114612:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa4700 x1876702067798528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764259.114634:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067798528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114645:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764259.114653:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764259.114674:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114684:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa5880 x1876702067798656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764259.114695:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067798656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114703:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114707:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114710:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764259.114716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764259.114733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114742:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa4a80 x1876702067798784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764259.114765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067798784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114773:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067798784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764259.114819:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764259.114839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114849:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa4380 x1876702067798912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764259.114874:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067798912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114882:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067798912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764259.114895:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764259.114913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764259.114923:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764259/real 1789764259] req@ffff936010aa5180 x1876702067799040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764259.114934:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067799040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764259.114948:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764259.114951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764259.114954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799040:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764259.883250:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764260.907776:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764261.930041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764262.954087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764263.978070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764264.234091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764264.234104:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234116:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234127:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234133:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764264.234166:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067799168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764264.234196:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234199:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234201:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234205:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234213:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764264.234228:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067799296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764264.234254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789764264.234263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067799168:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764264.234267:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234279:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234286:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234291:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764264.234318:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067799424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234329:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764264.234335:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234340:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234344:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234348:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234353:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764264.234364:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067799552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234375:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764264.234381:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234385:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234388:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234393:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234398:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764264.234409:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067799680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234418:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764264.234425:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234429:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234435:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234440:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234445:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764264.234456:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067799808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764264.234463:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764264.234466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764264.234469:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764264.234477:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1732 00000100:00080000:0.0:1789764264.234481:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764264.234484:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764264.234490:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764264.234492:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764264.234507:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067799936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764264.234713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764264.234742:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764264.234762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764264.234782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764264.234839:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799936:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764265.001992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764266.026150:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764267.050064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789764267.306239:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789764267.306305:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764267.306330:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306377:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306388:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306395:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306403:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306411:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764267.306418:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764267.306935:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020dac380 x1876702067799424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.306981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067799424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.306993:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.306997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307002:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764267.307010:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020daf100 x1876702067799552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307029:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067799552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307036:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764267.307050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020dac700 x1876702067799680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307061:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067799680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764267.307695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020dafb80 x1876702067799808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067799808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307725:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764267.307731:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020dac000 x1876702067799936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067799936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067799936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764267.307763:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020dae680 x1876702067799168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307773:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067799168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307784:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067799168:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764267.307835:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764264/real 1789764267] req@ffff936020daf480 x1876702067799296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764267.307853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067799296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764289 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764267.307862:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764267.307865:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764267.307870:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067799296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764268.074963:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764269.099273:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764269.357872:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.357883:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764269.357889:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.357901:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.357917:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.357925:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764269.357974:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067800064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764269.358075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067800064:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764269.358154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764269.358421:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.358426:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764269.358428:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.358433:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.358438:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.358443:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.358454:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764269.358469:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067800192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764269.362155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa6300 x1876702067800064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.362193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067800064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.362207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.362212:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.362217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067800064:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764269.362268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764269.362836:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.362864:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa7800 x1876702067800192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.362880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067800192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.362891:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.362896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.362900:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800192:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764269.363026:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.363032:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764269.363035:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.363042:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.363049:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.363057:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.363074:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764269.363107:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067800320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764269.363123:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.363126:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764269.363128:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.363132:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.363136:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.363140:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.363146:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764269.363158:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067800448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764269.363167:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.363170:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764269.363172:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.363175:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.363179:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.363182:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.363187:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764269.363197:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067800576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764269.363206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.363210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764269.363212:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.363216:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.363220:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.363223:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.363228:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764269.363238:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067800704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764269.363247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764269.363256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764269.363258:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764269.363262:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1737 00000100:00080000:3.0:1789764269.363265:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764269.363268:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764269.363273:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764269.363283:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067800832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764269.363444:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764269.363482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.363499:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa7b80 x1876702067800320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.363514:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067800320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.363522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.363526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.363529:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764269.363535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764269.363557:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.363569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa7800 x1876702067800448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.363580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067800448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.363589:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.363592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.363595:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764269.363601:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764269.363621:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.363633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa6300 x1876702067800576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.363645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067800576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.363652:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.363656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.363659:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067800576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764269.363665:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067800704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764269.363684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.363709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa5180 x1876702067800704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.363721:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067800704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.363729:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.363733:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.363735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067800704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764269.363742:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067800832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764269.363763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764269.363774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764269/real 1789764269] req@ffff936010aa4380 x1876702067800832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764269.363832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067800832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764294 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764269.363840:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764269.363843:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764269.363846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067800832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764270.122053:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764271.146058:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764272.170084:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764273.193983:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764274.219938:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764274.473914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.473923:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764274.473927:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.473937:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.473948:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.473955:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764274.473999:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067800960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764274.474035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764274.474041:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.474045:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474049:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474053:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474065:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764274.474078:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067801088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764274.474087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764274.474092:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.474096:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474099:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474102:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764274.474116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067801216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764274.474125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764274.474131:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.474135:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474138:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474142:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764274.474170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067801344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764274.474181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764274.474188:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789764274.474189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067800960:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764274.474195:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474201:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474205:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474210:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764274.474222:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067801472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764274.474232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764274.474237:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.474242:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474246:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474250:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474254:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764274.474268:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067801600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764274.474273:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764274.474287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764274.474291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764274.474293:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764274.474299:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1742 00000100:00080000:3.0:1789764274.474303:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764274.474307:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764274.474311:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764274.474325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067801728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764274.474330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa4a80 x1876702067800960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.474357:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067800960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.474369:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.474374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.474379:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067800960:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764274.474386:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764274.474406:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.474418:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa5880 x1876702067801088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.474429:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067801088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.474439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.474442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.474446:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764274.474453:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764274.474474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.474485:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa4700 x1876702067801216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.474497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067801216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.474505:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.474508:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.474511:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764274.474545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764274.474565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.474576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa7100 x1876702067801344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.474587:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067801344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.474595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.474598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.474602:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764274.474608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764274.474919:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.474935:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa4000 x1876702067801472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.474947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067801472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.474955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.474958:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.474962:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764274.474968:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067801600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764274.474988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.474997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa5c00 x1876702067801600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.475009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067801600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.475017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.475021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.475024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067801600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764274.475030:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067801728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764274.475049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764274.475059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764274/real 1789764274] req@ffff936010aa6a00 x1876702067801728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764274.475070:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067801728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764274.475078:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764274.475081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764274.475084:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067801728:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764275.242040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764276.266174:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764277.290542:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764278.314769:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764279.338365:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764279.593988:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594000:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764279.594005:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594017:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594031:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594041:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764279.594089:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067801856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594126:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594130:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764279.594132:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594137:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594141:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594145:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764279.594182:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067801984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594192:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594196:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764279.594197:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594202:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594206:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594209:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594214:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764279.594226:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067802112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594235:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594238:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764279.594240:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594244:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594248:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594251:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594256:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764279.594266:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067802240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594275:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594278:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764279.594280:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594283:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594286:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594290:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594295:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764279.594305:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067802368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594314:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594317:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764279.594319:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594322:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594326:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594330:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594334:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764279.594346:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067802496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764279.594355:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764279.594359:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764279.594360:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764279.594364:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1747 00000100:00080000:3.0:1789764279.594367:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764279.594370:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764279.594374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764279.594383:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067802624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764279.594582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067801856:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764279.594661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.594728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa6680 x1876702067801856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.594746:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067801856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.594757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.594762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.594766:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067801856:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764279.594774:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764279.594829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.594844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa5f80 x1876702067801984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.594862:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067801984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.594870:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.594873:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.594877:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067801984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764279.594883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764279.594904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.594978:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa5500 x1876702067802112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.594990:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067802112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.594997:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.595000:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.595003:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764279.595009:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764279.595028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.595037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa4e00 x1876702067802240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.595048:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067802240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.595062:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.595065:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.595068:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764279.595073:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764279.595092:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.595103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa7480 x1876702067802368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.595114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067802368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.595122:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.595125:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.595129:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764279.595134:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067802496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764279.595153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.595275:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa6a00 x1876702067802496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.595292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067802496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.595300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.595303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.595313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067802496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764279.595320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067802624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764279.595343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764279.595356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764279/real 1789764279] req@ffff936010aa5c00 x1876702067802624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764279.595368:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067802624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764279.595375:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764279.595378:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764279.595382:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067802624:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764280.362675:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764281.387780:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764282.410011:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764283.434632:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764284.458437:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764284.714481:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.714493:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764284.714498:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.714511:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.714527:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.714536:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764284.714586:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067802752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764284.714686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067802752:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764284.714785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764284.714874:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.714878:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764284.714880:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.714885:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.714891:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.714895:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.714907:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764284.714920:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067802880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764284.715903:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020daed80 x1876702067802752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.715926:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067802752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.715938:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.715943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.715947:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067802752:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764284.715979:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764284.716019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.716034:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020dad880 x1876702067802880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.716051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067802880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.716058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.716061:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.716064:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067802880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764284.718565:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.718576:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764284.718582:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.718593:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.718606:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.718617:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.718634:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764284.718691:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067803008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764284.718712:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.718715:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764284.718717:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.718721:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.718724:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.718728:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.718733:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764284.718745:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067803136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764284.718754:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.718757:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764284.718758:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.718762:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.718765:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.718769:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.718773:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764284.718784:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067803264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764284.718793:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.718796:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764284.718815:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.718819:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.718822:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.718826:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.718830:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764284.718841:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067803392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764284.718849:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764284.718852:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764284.718854:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764284.718858:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1752 00000100:00080000:0.0:1789764284.718861:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764284.718864:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764284.718868:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764284.718876:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067803520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764284.719147:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764284.719226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.719263:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020dad500 x1876702067803008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.719445:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067803008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.719459:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.719463:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.719467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764284.719475:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764284.719501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.719513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020dad880 x1876702067803136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.719526:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067803136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.719534:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.719538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.719541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764284.719546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764284.719566:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.719577:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020daed80 x1876702067803264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.719590:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067803264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.719598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.719601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.719605:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764284.719612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067803392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764284.719630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.719641:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020daf480 x1876702067803392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.719651:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067803392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.719665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.719679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.719683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067803392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764284.719688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067803520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764284.719708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764284.719719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764284/real 1789764284] req@ffff936020dae680 x1876702067803520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764284.719729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067803520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764284.719737:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764284.719740:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764284.719743:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067803520:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764285.482051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764286.506311:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764287.530212:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764288.556211:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764289.578184:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764289.833928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.833940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764289.833943:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.833955:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.833967:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.833974:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764289.834016:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067803648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834051:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834055:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764289.834056:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834060:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834064:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834067:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834081:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764289.834093:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067803776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764289.834105:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834108:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834111:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834124:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834128:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764289.834137:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067803904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764289.834153:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834157:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834160:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834163:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834167:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764289.834174:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067804032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764289.834186:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834188:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834198:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764289.834207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067804160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764289.834219:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834222:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834225:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834228:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764289.834241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067804288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764289.834248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764289.834250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764289.834252:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764289.834255:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1757 00000100:00080000:0.0:1789764289.834258:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764289.834261:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764289.834265:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764289.834273:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067804416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764289.834795:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067803648:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764289.834875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.834906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dac000 x1876702067803648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.834917:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067803648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.834931:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.834935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.834939:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067803648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764289.834947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764289.834967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.834978:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dafb80 x1876702067803776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.834986:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067803776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.834991:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.834993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.834995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764289.834998:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764289.835606:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.835640:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dac700 x1876702067803904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.835653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067803904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.835661:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.835664:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.835667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067803904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764289.835674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764289.835694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.835704:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020daf100 x1876702067804032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.835714:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067804032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.835721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.835723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.835726:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764289.835730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764289.835748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.835757:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dac380 x1876702067804160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.835766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067804160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.835772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.835775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.835778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764289.835783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067804288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764289.836039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.836050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dad180 x1876702067804288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.836061:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067804288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.836068:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.836071:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.836074:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067804288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764289.836079:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067804416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764289.836097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764289.836108:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764289/real 1789764289] req@ffff936020dae300 x1876702067804416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764289.836118:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067804416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764289.836124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764289.836127:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764289.836130:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067804416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764290.602676:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764291.626186:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764292.650144:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764293.674021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764294.698033:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764294.953965:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.953977:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764294.953982:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.953994:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954008:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954017:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764294.954068:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067804544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954106:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954111:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764294.954113:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954117:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954121:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954125:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954136:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764294.954149:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067804672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954159:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954162:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764294.954164:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954167:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954171:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954174:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954179:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764294.954189:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067804800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954197:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954201:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764294.954203:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954206:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954209:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954214:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764294.954228:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067804928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954237:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954241:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764294.954242:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954245:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954249:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954253:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954258:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764294.954269:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067805056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954277:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954280:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764294.954281:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954285:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954288:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954291:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954296:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764294.954306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067805184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764294.954315:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764294.954318:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764294.954320:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764294.954324:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1762 00000100:00080000:0.0:1789764294.954327:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764294.954330:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764294.954334:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764294.954342:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067805312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764294.954718:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067804544:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764294.954844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.954910:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020dadf80 x1876702067804544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.954927:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067804544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.954935:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.954938:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.954941:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067804544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764294.954947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764294.954966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.954975:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020daea00 x1876702067804672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.954987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067804672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.954992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.954993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.954995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764294.954999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764294.955010:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.955016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020dace00 x1876702067804800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.955023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067804800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.955028:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.955030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.955032:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764294.955035:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764294.955046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.955051:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020dadc00 x1876702067804928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.955057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067804928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.955066:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.955068:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.955070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067804928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764294.955075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764294.955093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.955104:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020daf800 x1876702067805056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.955114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067805056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.955121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.955124:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.955127:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764294.955133:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067805184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764294.955151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.955161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020dae300 x1876702067805184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.955172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067805184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.955180:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.955183:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.955187:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067805184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764294.955192:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067805312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764294.955209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764294.955217:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764294/real 1789764294] req@ffff936020dad180 x1876702067805312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764294.955228:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067805312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764294.955235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764294.955237:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764294.955241:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067805312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764295.722298:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764296.746777:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764297.769952:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764298.795831:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764299.819080:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764300.074432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764300.074445:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074455:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074466:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074472:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764300.074505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067805440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074544:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764300.074546:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074550:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074553:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074577:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764300.074589:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067805568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074602:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764300.074603:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074607:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074611:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074614:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074618:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764300.074629:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067805696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764300.074642:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074645:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074648:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074651:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074655:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764300.074664:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067805824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764300.074677:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074682:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074686:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074690:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764300.074707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067805952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074717:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764300.074722:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074726:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074730:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764300.074748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067806080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764300.074758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764300.074761:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764300.074762:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764300.074766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1767 00000100:00080000:1.0:1789764300.074770:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764300.074773:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764300.074776:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764300.074795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067806208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764300.075221:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067805440:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764300.075327:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef6300 x1876702067805440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075428:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067805440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075442:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075447:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075453:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067805440:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764300.075463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764300.075500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075515:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef6d80 x1876702067805568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067805568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075544:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075548:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764300.075575:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764300.075603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075622:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef4e00 x1876702067805696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075639:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067805696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764300.075678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764300.075702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef7100 x1876702067805824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067805824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764300.075749:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764300.075784:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075797:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef4380 x1876702067805952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075858:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067805952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075883:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075887:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067805952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764300.075894:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764300.075918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.075931:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef4000 x1876702067806080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.075943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067806080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.075951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.075954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.075958:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764300.075964:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764300.076000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764300.076013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764300/real 1789764300] req@ffff936010ef5c00 x1876702067806208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764300.076025:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067806208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764300.076040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764300.076044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764300.076047:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764300.842847:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764301.865981:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764302.891663:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764303.914092:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764304.938227:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764305.194405:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.194418:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764305.194423:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.194435:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.194448:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.194545:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764305.194601:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067806336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764305.194684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067806336:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764305.194767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764305.194827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.194832:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764305.194834:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.194840:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.194844:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.194849:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.194862:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764305.194876:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067806464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764305.197333:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.197343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764305.197347:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.197357:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.197367:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.197375:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.197388:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764305.197424:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067806592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764305.197439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.197442:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764305.197444:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.197449:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.197453:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.197457:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.197461:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764305.197473:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067806720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764305.197483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.197486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764305.197488:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.197492:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.197495:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.197498:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.197503:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764305.197514:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067806848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764305.198001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef5880 x1876702067806336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.198037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067806336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.198059:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.198064:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.198068:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067806336:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764305.198467:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764305.198604:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.198626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef4700 x1876702067806464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.198640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067806464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.198648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.198652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.198655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764305.199001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764305.199037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.199059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef5180 x1876702067806592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.199081:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067806592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.199130:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.199135:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.199140:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764305.199147:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764305.199173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.199185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef4a80 x1876702067806720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.199198:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067806720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.199206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.199209:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.199212:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806720:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764305.201652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.201659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764305.201662:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.201673:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.201684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.201693:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.201707:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764305.201744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067806976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764305.201761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764305.201764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764305.201767:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764305.201770:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1773 00000100:00080000:1.0:1789764305.201773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764305.201777:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764305.201781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764305.201792:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067807104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764305.202010:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764305.202088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.202121:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef5c00 x1876702067806848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.202140:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067806848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.202151:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.202155:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.202159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067806848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764305.202166:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764305.202191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.202203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef4000 x1876702067806976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.202217:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067806976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.202225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.202228:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.202231:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067806976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764305.202237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067807104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764305.202257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764305.202268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764305/real 1789764305] req@ffff936010ef4380 x1876702067807104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764305.202279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067807104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764305.202291:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764305.202294:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764305.202297:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067807104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764305.962148:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764306.986219:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764308.010092:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764309.034026:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764310.059610:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764310.315721:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.315732:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764310.315737:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.315748:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.315761:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.315769:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764310.315839:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067807232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764310.315931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067807232:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764310.316005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764310.316035:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.316042:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764310.316043:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.316047:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.316051:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.316055:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.316065:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764310.316078:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067807360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764310.316315:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa4000 x1876702067807232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.316336:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067807232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.316347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.316351:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.316364:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067807232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764310.316396:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764310.316612:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.316632:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa7100 x1876702067807360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764310.316688:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.316693:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764310.316695:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.316700:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.316705:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.316709:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.316715:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764310.316729:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067807488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764310.316739:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.316742:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764310.316744:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.316747:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.316750:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.316754:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.316758:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764310.318096:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067807360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.318113:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.318118:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.318122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764310.318149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764310.318197:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.318818:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa4700 x1876702067807488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.318840:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067807488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.318849:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.318853:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.318856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764310.318986:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067807616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764310.319013:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.319018:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764310.319020:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.319025:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.319031:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.319038:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.319048:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764310.319066:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067807744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764310.319074:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.319077:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764310.319078:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.319081:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.319084:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.319087:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.319092:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764310.319099:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067807872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764310.319107:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764310.319110:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764310.319111:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764310.319115:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1778 00000100:00080000:3.0:1789764310.319117:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764310.319121:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764310.319125:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764310.319134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067808000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764310.319756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764310.319819:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.319839:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa5880 x1876702067807616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.319854:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067807616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.319863:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.319867:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.319870:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764310.319877:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764310.319897:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.319908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa4a80 x1876702067807744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.320123:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067807744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.320131:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.320135:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.320143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067807744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764310.320149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067807872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764310.320172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.320185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa4380 x1876702067807872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.320195:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067807872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.320203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.320876:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.320883:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067807872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764310.320891:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764310.321077:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764310.321092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764310/real 1789764310] req@ffff936010aa5180 x1876702067808000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764310.321105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067808000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764335 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764310.321119:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764310.321122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764310.321126:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764311.081954:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764312.106038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764313.130007:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764314.155704:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764315.178183:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764315.434039:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.434048:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764315.434053:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.434063:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.434074:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764315.434081:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764315.434123:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067808128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764315.434210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067808128:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764315.434326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764315.434352:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.434357:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764315.434359:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.434363:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.434367:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764315.434372:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764315.434390:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764315.434406:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067808256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764315.434526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef7100 x1876702067808128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764315.434542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067808128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764315.434551:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.434554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.434557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067808128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764315.434577:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764315.434595:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764315.434603:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef4e00 x1876702067808256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764315.434610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067808256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764315.434616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.434618:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.434620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764315.434693:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.434696:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764315.434698:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.434702:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.434705:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764315.434708:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764315.434712:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764315.434724:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067808384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764315.434752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764315.434776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764315.434791:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.434794:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764315.434795:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.434825:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.434828:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764315.434831:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764315.434836:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764315.434847:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067808512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764315.434985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef6d80 x1876702067808384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764315.434999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067808384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764315.435007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.435010:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.435013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764315.435027:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764315.435051:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764315.435068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef6300 x1876702067808512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764315.435071:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.435075:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764315.435077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764315.435080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067808512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764315.435081:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.435085:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764315.435087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764315.435090:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764315.435090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.435093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808512:192.168.203.137@tcp:8:kworker.0 00000008:00100000:1.0:1789764315.435096:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764315.435108:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067808640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764315.435121:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.435125:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764315.435127:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.435131:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.435134:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764315.435134:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808640:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764315.435138:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764315.435142:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764315.435152:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067808768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764315.435154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764315.435161:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764315.435165:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789764315.435165:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef7b80 x1876702067808640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764315.435167:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764315.435171:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1783 00000100:00080000:1.0:1789764315.435174:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764315.435177:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764315.435177:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067808640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789764315.435182:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789764315.435185:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.435188:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764315.435190:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067808896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764315.435191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067808640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764315.435200:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764315.435218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764315.435229:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef7480 x1876702067808768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764315.435241:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067808768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764315.435249:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.435252:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.435255:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764315.435260:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764315.435351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764315.435364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764315/real 1789764315] req@ffff936010ef7800 x1876702067808896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764315.435375:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067808896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764315.435462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764315.435466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764315.435469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067808896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764316.203425:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764317.227649:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764318.250165:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764319.274099:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764320.298866:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764320.554034:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.554044:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764320.554049:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.554061:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.554076:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.554085:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764320.554134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067809024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764320.554217:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067809024:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764320.554295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764320.554348:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.554352:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764320.554354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.554359:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.554364:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.554369:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.554380:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764320.554395:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067809152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764320.557048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061de680 x1876702067809024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.557079:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067809024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.557092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.557097:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.557101:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067809024:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764320.557135:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764320.557201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.557220:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061dc000 x1876702067809152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.557238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067809152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.557246:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.557249:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.557253:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809152:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764320.557518:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.557524:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764320.557527:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.557533:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.557539:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.557546:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.557702:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764320.557730:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067809280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764320.557745:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.557749:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764320.557751:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.557831:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.557836:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.557841:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.557846:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764320.557858:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067809408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764320.557866:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.557869:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764320.557871:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.557874:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.557878:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.557881:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.557886:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764320.557898:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067809536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764320.557907:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.557910:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764320.557911:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.557915:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.557918:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.557922:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.557926:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764320.557935:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067809664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764320.557943:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764320.557946:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764320.557947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764320.557951:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1788 00000100:00080000:2.0:1789764320.557954:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764320.557957:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764320.557960:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764320.557969:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067809792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764320.558048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764320.558076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.558090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061dd180 x1876702067809280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.558103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067809280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.558111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.558114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.558118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764320.558124:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764320.558144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.558155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061dce00 x1876702067809408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.558165:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067809408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.558178:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.558181:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.558185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764320.558190:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764320.558210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.558222:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061dc000 x1876702067809536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.558234:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067809536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.558241:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.558244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.558247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067809536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764320.558253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067809664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764320.558272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.558283:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061de680 x1876702067809664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.558294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067809664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.558533:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.558539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.558544:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067809664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764320.558847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067809792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764320.558874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764320.558888:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764320/real 1789764320] req@ffff9360061dfb80 x1876702067809792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764320.558901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067809792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764320.558909:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764320.558912:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764320.558915:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067809792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764321.327149:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764322.346073:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764323.370003:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764324.394497:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764325.419239:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764325.675257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675268:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764325.675273:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675286:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675299:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675307:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764325.675354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067809920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675392:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675396:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764325.675398:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675402:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675405:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675409:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675420:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764325.675431:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067810048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675443:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764325.675448:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675452:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675455:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675458:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675462:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764325.675471:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067810176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675479:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764325.675484:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675488:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675491:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675494:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675498:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764325.675506:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067810304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764325.675523:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675526:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675528:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675531:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675534:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764325.675544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067810432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675551:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764325.675556:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675559:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675562:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675565:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675569:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764325.675576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067810560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764325.675584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764325.675587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764325.675589:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764325.675594:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1793 00000100:00080000:1.0:1789764325.675598:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764325.675988:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764325.675998:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764325.676017:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067810688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764325.676677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067809920:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764325.676756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.676835:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef6680 x1876702067809920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.676856:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067809920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.676866:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.676870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.676874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067809920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764325.676883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764325.676909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.676920:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef5f80 x1876702067810048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.676931:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067810048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.676939:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.676942:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.676945:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764325.676951:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764325.676968:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.676976:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef6a00 x1876702067810176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.676986:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067810176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.676992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.676995:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.676998:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764325.677004:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764325.677021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.677031:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef5500 x1876702067810304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.677047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067810304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.677055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.677057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.677060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764325.677066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764325.677084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.677095:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef7800 x1876702067810432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.677105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067810432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.677112:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.677115:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.677118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764325.677123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067810560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764325.679003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.679033:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef7480 x1876702067810560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.679623:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067810560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.679642:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.679646:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.679650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067810560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764325.679661:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067810688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764325.679701:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764325.679713:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764325/real 1789764325] req@ffff936010ef7b80 x1876702067810688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764325.679725:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067810688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764325.679733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764325.679736:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764325.679739:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067810688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764326.442899:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764327.466070:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764328.490084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764329.514061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764330.538594:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764330.793973:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.793985:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764330.793991:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794004:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794018:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794028:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764330.794080:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067810816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794119:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794123:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764330.794125:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794130:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794134:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794138:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794149:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764330.794162:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067810944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794173:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764330.794178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794182:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794185:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794189:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764330.794205:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067811072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794221:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764330.794226:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794230:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794234:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794237:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764330.794252:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067811200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794260:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794263:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764330.794265:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794268:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794272:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794275:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794280:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764330.794290:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067811328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794300:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794303:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764330.794305:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794308:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794312:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794316:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794320:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764330.794341:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067811456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764330.794350:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764330.794353:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764330.794355:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764330.794359:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1798 00000100:00080000:2.0:1789764330.794363:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764330.794366:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764330.794371:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764330.794381:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067811584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764330.795280:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067810816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764330.795419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764330.795722:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764330.795758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764330.795783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764330.795855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067811456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764330.795881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067811584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764331.564976:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764332.586313:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764333.610739:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764333.866279:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764333.866343:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764333.872670:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764333.872910:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764333.872925:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764333.872936:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061df800 x1876702067810816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789764333.872936:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764333.872948:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764333.872959:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764333.872969:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764333.872978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067810816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.872994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.872999:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873004:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067810816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764333.873013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061dca80 x1876702067810944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067810944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873048:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873052:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873055:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067810944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764333.873065:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061dd500 x1876702067811072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873078:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067811072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873086:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764333.873099:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061df480 x1876702067811200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067811200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873121:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873124:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764333.873130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061df100 x1876702067811328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873140:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067811328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873147:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873150:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873153:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764333.873160:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061dd880 x1876702067811456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873173:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067811456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873180:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873183:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873187:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067811456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764333.873192:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764330/real 1789764333] req@ffff9360061ddc00 x1876702067811584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764333.873203:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067811584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764333.873210:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764333.873214:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764333.873217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067811584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764334.634411:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764335.658233:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764335.921089:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.921101:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764335.921106:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.921118:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.921132:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.921140:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764335.921185:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067811712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764335.921320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067811712:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764335.921416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764335.921447:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.921452:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764335.921454:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.921458:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.921463:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.921467:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.921481:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764335.921496:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067811840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764335.921867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef6300 x1876702067811712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.921888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067811712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.921899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.921904:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.921908:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067811712:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764335.921932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764335.921956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.921968:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef6d80 x1876702067811840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.921979:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067811840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.921987:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.921990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.921993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811840:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764335.922171:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.922176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764335.922179:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.922191:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.922196:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.922200:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.922206:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764335.922220:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067811968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764335.922230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.922234:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764335.922236:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.922240:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.922243:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.922246:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.922251:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764335.922261:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067812096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764335.922270:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.922273:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764335.922276:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.922279:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.922282:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.922286:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.922291:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764335.922309:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067812224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764335.922318:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.922321:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764335.922323:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.922327:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.922331:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.922334:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.922350:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764335.922362:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067812352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764335.922371:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764335.922374:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764335.922376:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764335.922379:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1803 00000100:00080000:1.0:1789764335.922383:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764335.922386:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764335.922400:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764335.922412:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067812480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764335.922757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764335.922794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.922852:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef4e00 x1876702067811968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.922866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067811968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.922876:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.922879:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.922883:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067811968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764335.922890:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764335.922913:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.922924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef7100 x1876702067812096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.922935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067812096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.922944:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.922947:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.922950:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764335.922957:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764335.922977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.922988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef4380 x1876702067812224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.922999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067812224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.923007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.923010:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.923013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764335.923019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067812352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764335.923039:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.923051:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef4000 x1876702067812352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.923071:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067812352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.923079:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.923082:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.923085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067812352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764335.923091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067812480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764335.923110:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764335.923121:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764335/real 1789764335] req@ffff936010ef5c00 x1876702067812480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764335.923133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067812480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764335.923139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764335.923142:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764335.923145:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067812480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764336.686883:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764337.706167:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764338.732818:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764339.754605:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764340.777972:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764341.033914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.033923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764341.033927:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.033936:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.033947:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.033954:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764341.034005:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067812608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764341.034031:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.034033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764341.034035:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.034038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.034041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.034044:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034055:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764341.034073:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067812608:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764341.034079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067812736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764341.034092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.034095:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764341.034096:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.034099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.034103:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.034106:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034110:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764341.034120:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067812864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764341.034127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.034130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764341.034130:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.034135:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1808 00000800:00000100:0.0:1789764341.034136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764341.034138:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.034142:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034145:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764341.034153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067812992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764341.034159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.034161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764341.034162:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.034164:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.034166:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.034168:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034171:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764341.034178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067813120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764341.034182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061dc380 x1876702067812608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764341.034188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764341.034194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764341.034198:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764341.034202:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.034205:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764341.034209:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067812608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764341.034209:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034215:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764341.034226:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067813248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764341.034230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764341.034233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764341.034236:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789764341.034238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764341.034241:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764341.034243:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067812608:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764341.034247:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1808 00000100:00080000:2.0:1789764341.034252:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764341.034266:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764341.034272:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764341.034280:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764341.034286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067813376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764341.034306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034317:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061dc700 x1876702067812736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034325:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067812736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034331:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034335:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764341.034339:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764341.034349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034363:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061ddf80 x1876702067812864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067812864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034377:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034379:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034381:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764341.034385:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764341.034396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034402:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061de300 x1876702067812992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067812992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034415:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034417:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067812992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764341.034420:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764341.034429:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034437:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061ddc00 x1876702067813120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034447:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067813120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034455:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034458:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034461:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764341.034467:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067813248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764341.034486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034497:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061dd880 x1876702067813248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067813248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067813248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764341.034541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067813376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764341.034560:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764341.034570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764341/real 1789764341] req@ffff9360061df100 x1876702067813376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764341.034581:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067813376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764341.034589:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764341.034592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764341.034596:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067813376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764341.801977:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764342.825999:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764343.850347:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764344.874041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764345.897974:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764346.153935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.153951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764346.153956:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.153968:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.153983:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.153991:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764346.154034:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067813504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764346.154065:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154068:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154071:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154082:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764346.154090:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067813632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764346.154101:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154104:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154106:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154111:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764346.154117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067813760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764346.154128:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154131:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154133:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154135:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154139:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764346.154149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067813504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764346.154149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067813888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764346.154183:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154190:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154193:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154198:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154203:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764346.154230:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067814016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154238:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154241:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764346.154242:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154245:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154247:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154252:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789764346.154258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154260:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764346.154272:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067814144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764346.154279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764346.154282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764346.154284:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764346.154286:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1813 00000100:00080000:2.0:1789764346.154288:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764346.154290:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764346.154294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764346.154305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067814272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764346.154326:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061df480 x1876702067813504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154350:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067813504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154368:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067813504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764346.154385:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764346.154410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061dd500 x1876702067813632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154433:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067813632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154441:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154445:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764346.154454:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764346.154485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061dca80 x1876702067813760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067813760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764346.154549:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764346.154570:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061df800 x1876702067813888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154594:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067813888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154602:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154605:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067813888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764346.154615:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764346.154634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061dfb80 x1876702067814016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067814016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764346.154674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067814144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764346.154694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154704:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061de680 x1876702067814144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154715:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067814144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154723:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154726:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067814144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764346.154736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067814272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764346.154753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764346.154764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764346/real 1789764346] req@ffff9360061dc000 x1876702067814272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764346.154775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067814272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764346.154783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764346.154786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764346.154789:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067814272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764346.921988:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764347.946313:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764348.970252:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764349.994000:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764351.017950:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764351.274019:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274031:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764351.274035:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274045:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274055:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274062:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764351.274097:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067814400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764351.274137:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274142:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274149:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274157:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764351.274169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067814528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274178:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764351.274191:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274195:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274198:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274201:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274205:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764351.274215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067814656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764351.274229:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274232:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274235:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274238:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274241:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764351.274252:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067814784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274261:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764351.274265:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274269:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274272:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274275:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764351.274289:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067814400:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764351.274290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067814912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764351.274308:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274311:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274315:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274318:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274323:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764351.274334:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067815040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764351.274343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764351.274346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1789764351.274350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764351.274351:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764351.274357:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1819 00000100:00080000:3.0:1789764351.274361:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764351.274365:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764351.274370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764351.274382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067815168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764351.274413:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa6300 x1876702067814400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274431:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067814400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274447:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274451:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274454:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067814400:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764351.274461:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764351.274480:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274490:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa7800 x1876702067814528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274501:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067814528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764351.274520:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764351.274537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274547:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa7b80 x1876702067814656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067814656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274569:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764351.274578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764351.274593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa5180 x1876702067814784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274612:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067814784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764351.274652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764351.274666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274675:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa4380 x1876702067814912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274685:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067814912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274693:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274696:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274699:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067814912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764351.274704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764351.274717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa4a80 x1876702067815040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274736:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067815040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274743:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274746:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764351.274753:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764351.274767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764351.274787:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764351/real 1789764351] req@ffff936010aa5880 x1876702067815168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764351.274833:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067815168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764376 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764351.274841:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764351.274850:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764351.274853:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764352.043000:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764353.066000:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764354.090015:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764355.114020:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764356.138380:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764356.393940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.393948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764356.393952:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.393962:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.393974:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.393981:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764356.394022:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067815296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764356.394050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764356.394054:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394056:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394059:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.394061:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394070:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764356.394077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067815424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764356.394085:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394087:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764356.394088:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.394098:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394104:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764356.394113:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067815552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764356.394118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067815296:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764356.394128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764356.394153:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394157:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394160:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.394164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394170:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764356.394183:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067815680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764356.394193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764356.394198:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394202:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394206:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789764356.394211:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764356.394212:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764356.394232:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067815808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764356.394241:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764356.394247:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394250:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394254:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.394258:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394263:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764356.394275:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067815936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764356.394282:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764356.394286:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789764356.394287:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa4700 x1876702067815296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764356.394288:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764356.394294:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1824 00000100:00080000:3.0:1789764356.394299:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764356.394304:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764356.394310:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764356.394321:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067815296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764356.394323:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067816064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764356.394359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394364:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394368:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067815296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764356.394374:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764356.394407:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394421:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa7100 x1876702067815424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394434:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067815424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394441:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394445:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764356.394479:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764356.394500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa4000 x1876702067815552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394530:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067815552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764356.394554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764356.394574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa5c00 x1876702067815680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067815680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764356.394618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764356.394633:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394641:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa6a00 x1876702067815808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394648:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067815808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067815808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764356.394667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764356.394686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa7480 x1876702067815936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067815936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394719:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394722:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394725:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067815936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764356.394730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764356.394750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764356.394760:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764356/real 1789764356] req@ffff936010aa4e00 x1876702067816064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764356.394785:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067816064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764381 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764356.394793:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764356.394796:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764356.394826:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764357.162023:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764358.185956:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764359.210033:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764360.235501:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764361.258071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764361.514220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764361.514236:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514248:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514262:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.514271:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764361.514321:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067816192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.514358:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764361.514364:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514369:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514372:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.514375:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.514386:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764361.514399:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067816320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.514413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764361.514418:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514422:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514426:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.514429:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.514434:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764361.514447:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067816448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.514455:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764361.514460:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514463:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514467:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.514471:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.514475:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764361.514486:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067816576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.514495:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764361.514499:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514503:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514506:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.514510:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.514515:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764361.514524:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067816704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.514533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.514537:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764361.514539:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.514542:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.514545:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764361.514728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067816192:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764361.514838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.514903:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061dce00 x1876702067816192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.514924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067816192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.514939:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.514943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.514947:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067816192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764361.514954:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764361.514978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.514989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061dd180 x1876702067816320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.515000:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067816320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.515008:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.515011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.515014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764361.515019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764361.515037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.515048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061dc000 x1876702067816448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.515058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067816448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.515066:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.515070:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.515073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764361.515079:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764361.515098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.515119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061de680 x1876702067816576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.515130:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067816576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.515137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.515141:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.515144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764361.515151:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764361.515170:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.515180:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061dfb80 x1876702067816704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.515191:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067816704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.515199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.515202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.515205:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067816704:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789764361.515362:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.515370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764361.515385:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067816832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764361.515404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764361.515408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764361.515410:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764361.515414:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1829 00000100:00080000:2.0:1789764361.515419:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764361.515422:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764361.515427:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764361.515437:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067816960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764361.515984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764361.516023:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.516040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061df800 x1876702067816832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.516055:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067816832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.516065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.516068:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.516072:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764361.516079:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764361.516101:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764361.516124:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764361/real 1789764361] req@ffff9360061dca80 x1876702067816960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764361.516137:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067816960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764361.516144:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764361.516146:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764361.516149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067816960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764362.282027:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764363.308300:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764364.330759:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764365.353919:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764366.378537:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764366.635828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.635840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764366.635845:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.635858:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.635872:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.635881:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764366.635933:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067817088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.635964:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.635968:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764366.635970:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.635975:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.635979:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.635983:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.635994:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764366.636007:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067817216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.636017:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.636021:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764366.636023:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.636027:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.636030:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.636034:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.636039:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764366.636051:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067817344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.636062:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.636065:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764366.636067:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.636083:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.636087:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.636091:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.636096:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764366.636109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067817472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.636119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.636122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764366.636124:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.636127:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.636131:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.636134:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.636139:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764366.636149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067817600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.636158:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.636162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764366.636164:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.636168:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.636171:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.636175:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.636181:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764366.636193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067817728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764366.636202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764366.636205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764366.636208:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764366.636211:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1834 00000100:00080000:2.0:1789764366.636215:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764366.636219:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764366.636224:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764366.636233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067817856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764366.638457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067817088:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764366.638533:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.638588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061dd500 x1876702067817088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.638608:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067817088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.638625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.638630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.638634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067817088:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764366.638642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764366.638678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.638691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061df480 x1876702067817216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.638705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067817216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.638713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.638716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.638719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764366.638725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764366.638744:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.638754:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061df100 x1876702067817344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.638766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067817344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.638775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.638778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.638781:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764366.638958:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764366.638983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.638995:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061dd880 x1876702067817472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.639006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067817472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.639013:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.639016:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.639019:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764366.639025:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764366.639043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.639053:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061ddc00 x1876702067817600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.639085:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067817600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.639092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.639095:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.639098:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067817600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764366.639104:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067817728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764366.639123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.639135:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061de300 x1876702067817728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.639152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067817728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.639160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.639163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.639166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067817728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764366.639172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067817856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764366.639192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764366.639204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764366/real 1789764366] req@ffff9360061ddf80 x1876702067817856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764366.639216:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067817856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764366.639223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764366.639227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764366.639229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067817856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764367.401953:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764368.426737:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764369.450704:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764370.474439:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764371.502341:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764371.754298:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754307:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764371.754312:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754322:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754333:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754340:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764371.754374:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067817984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754409:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754413:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764371.754415:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754419:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754423:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754426:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754435:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764371.754447:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067818112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754457:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754461:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764371.754462:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754466:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754469:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754473:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754478:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764371.754488:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067818240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754504:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754507:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764371.754508:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754512:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754515:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754519:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754523:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764371.754533:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067818368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754541:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754543:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764371.754545:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754548:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754551:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754554:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754558:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764371.754569:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067818496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754578:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754581:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764371.754583:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754586:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754590:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754593:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754598:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764371.754608:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067818624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764371.754616:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764371.754619:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764371.754621:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764371.754624:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1839 00000100:00080000:1.0:1789764371.754628:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764371.754631:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764371.754635:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764371.754645:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067818752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764371.755121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067817984:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764371.755187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755223:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef4a80 x1876702067817984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755246:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067817984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755255:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755259:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755262:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067817984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764371.755268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764371.755301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755313:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef5180 x1876702067818112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755324:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067818112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755331:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755334:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755337:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764371.755342:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764371.755372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef4700 x1876702067818240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755394:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067818240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755410:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755413:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764371.755418:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764371.755435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755443:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef5880 x1876702067818368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755452:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067818368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755459:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764371.755469:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764371.755484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755493:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef5c00 x1876702067818496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755501:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067818496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755508:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755510:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755513:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067818496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764371.755518:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067818624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764371.755533:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755541:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef4000 x1876702067818624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067818624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755555:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067818624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764371.755565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067818752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764371.755592:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764371.755601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764371/real 1789764371] req@ffff936010ef4380 x1876702067818752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764371.755611:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067818752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764371.755617:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764371.755620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764371.755623:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067818752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764372.522520:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764373.546315:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764374.570422:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764375.593989:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764376.618972:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764376.874035:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874046:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764376.874051:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874062:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874074:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874083:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764376.874134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067818880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874170:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874173:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764376.874175:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874178:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874181:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874185:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764376.874208:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067819008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874217:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874221:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764376.874222:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874227:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874229:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874234:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874238:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764376.874249:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067819136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874258:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874261:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764376.874263:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874267:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874270:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874274:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874278:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764376.874291:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067819264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874299:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874303:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764376.874304:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874308:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874311:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874314:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874319:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764376.874329:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067819392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874338:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874342:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764376.874344:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874347:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874350:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874354:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874358:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764376.874367:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067819520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764376.874375:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764376.874378:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764376.874380:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764376.874383:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1844 00000100:00080000:3.0:1789764376.874386:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764376.874390:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764376.874394:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764376.874466:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067819648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764376.874780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067818880:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764376.874907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.874972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa5500 x1876702067818880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.874992:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067818880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875021:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875030:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067818880:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764376.875038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764376.875064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa5f80 x1876702067819008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.875091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067819008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875099:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875102:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875105:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764376.875111:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764376.875131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875143:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa6680 x1876702067819136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.875156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067819136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875168:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875171:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764376.875178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764376.875201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875211:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa4e00 x1876702067819264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.875222:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067819264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875231:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875234:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764376.875243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764376.875263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa7480 x1876702067819392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.875285:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067819392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875292:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764376.875304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067819520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764376.875322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875332:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa6a00 x1876702067819520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.875344:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067819520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.875352:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.875356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.875359:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067819520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764376.875365:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067819648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764376.875385:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764376.875396:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764376/real 1789764376] req@ffff936010aa5c00 x1876702067819648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764376.876178:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067819648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764376.876190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764376.876194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764376.876197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067819648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764377.642005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764378.666068:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764379.689933:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764380.714020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764381.738155:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764381.994950:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764381.994972:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764381.994978:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764381.994990:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764381.995005:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764381.995014:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764381.995061:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067819776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764381.995172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067819776:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764381.995262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764381.995345:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764381.995349:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764381.995361:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764381.995366:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764381.995370:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764381.995374:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764381.995386:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764381.995401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067819904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764381.995549:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764381/real 1789764381] req@ffff936010aa4000 x1876702067819776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764381.995572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067819776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764381.995590:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764381.995594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764381.995599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067819776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764381.995629:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764381.995655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764381.995668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764381/real 1789764381] req@ffff936010aa7100 x1876702067819904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764381.995680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067819904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764381.995688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764381.995691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764381.995694:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067819904:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764381.997996:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764381.998009:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764381.998013:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764381.998025:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764381.998039:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764381.998047:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764381.998062:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764381.998105:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067820032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764381.998310:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764381.998386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764381.998422:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764381.998426:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764381.998428:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764381.998433:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764381.998437:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764381.998441:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764381.998447:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764381.998461:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067820160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764382.007085:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764381/real 1789764381] req@ffff936010aa4700 x1876702067820032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764382.007120:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067820032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764382.007135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764382.007140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764382.007429:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764382.007567:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764382.007649:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764382.007671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764382/real 1789764382] req@ffff936010aa5880 x1876702067820160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764382.007685:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067820160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764382.007695:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764382.007699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764382.007703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820160:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764382.007807:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764382.007813:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764382.007817:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764382.007822:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764382.007829:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764382.007837:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764382.007850:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764382.007879:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067820288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764382.007894:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764382.007897:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764382.007899:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764382.007903:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764382.007907:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764382.007911:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764382.007917:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764382.007929:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067820416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764382.007938:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764382.007941:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764382.007942:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764382.007946:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1849 00000100:00080000:3.0:1789764382.007949:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764382.007952:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764382.007968:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764382.007979:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067820544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764382.008109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764382.008135:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764382.008258:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764382/real 1789764382] req@ffff936010aa4a80 x1876702067820288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764382.008271:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067820288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764382.008277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764382.008279:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764382.008281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764382.008286:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067820416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764382.008304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764382.008312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764382/real 1789764382] req@ffff936010aa4380 x1876702067820416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764382.008319:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067820416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764382.008323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764382.008325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764382.008327:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067820416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764382.008330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067820544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764382.008342:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764382.008348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764382/real 1789764382] req@ffff936010aa5180 x1876702067820544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764382.008354:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067820544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764382.008359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764382.008360:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764382.008362:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067820544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764382.762486:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764383.785990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764384.810075:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764385.834315:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764386.858615:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764387.113968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.113979:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764387.113985:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114008:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114023:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114032:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764387.114084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067820672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764387.114126:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114130:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114133:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114137:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114148:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764387.114162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067820800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764387.114177:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114181:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114188:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764387.114205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067820928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764387.114219:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114223:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114226:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114229:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114233:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764387.114242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067821056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764387.114255:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114258:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114261:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114264:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114268:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764387.114276:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067821184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764387.114289:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114292:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114295:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114299:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114303:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764387.114313:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067821312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764387.114322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764387.114325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764387.114327:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764387.114331:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1854 00000100:00080000:2.0:1789764387.114334:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764387.114337:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764387.114341:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764387.114350:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067821440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764387.114483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067820672:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764387.114561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.114614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061dc700 x1876702067820672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.114633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067820672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.114646:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.114649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.114653:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067820672:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764387.114660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764387.114683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.114708:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061dc380 x1876702067820800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.114720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067820800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.114728:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.114732:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.114735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764387.114741:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764387.114760:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.114772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061ddf80 x1876702067820928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.114782:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067820928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.114789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.114792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.114815:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067820928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764387.114821:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764387.114840:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.114861:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061de300 x1876702067821056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.114872:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067821056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.114880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.114884:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.114887:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764387.114892:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764387.114910:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.114931:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061ddc00 x1876702067821184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.114943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067821184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.114951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.114957:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.114959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764387.114966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067821312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764387.115048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.115063:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061dd880 x1876702067821312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.115077:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067821312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.115084:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.115087:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.115090:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067821312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764387.115096:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067821440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764387.115116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764387.115127:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764387/real 1789764387] req@ffff9360061df100 x1876702067821440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764387.115138:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067821440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764387.115146:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764387.115149:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764387.115152:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067821440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764387.885111:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764388.906253:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764389.931251:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764390.954167:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764391.978172:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764392.233982:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764392.234005:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234016:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234031:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234040:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764392.234088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067821568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764392.234132:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234144:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764392.234169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067821696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764392.234185:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234189:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234192:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234195:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234200:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764392.234210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067821824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764392.234224:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234228:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234232:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234236:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234240:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764392.234249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067821952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764392.234262:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234266:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234269:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234272:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764392.234284:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067822080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764392.234299:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234302:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234306:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234309:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234314:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764392.234323:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067822208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764392.234331:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764392.234335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764392.234336:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764392.234340:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1860 00000100:00080000:2.0:1789764392.234352:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764392.234357:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764392.234361:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764392.234372:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067822336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764392.234392:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067821568:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764392.234471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061df480 x1876702067821568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234540:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067821568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234552:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234562:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067821568:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764392.234569:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764392.234592:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234606:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061dd500 x1876702067821696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234618:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067821696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764392.234640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764392.234660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234672:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061dca80 x1876702067821824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067821824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764392.234711:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764392.234731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234742:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061df800 x1876702067821952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234754:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067821952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234761:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234764:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067821952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764392.234772:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764392.234790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061dfb80 x1876702067822080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234839:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067822080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234847:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234851:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764392.234861:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067822208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764392.234880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.234901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061de680 x1876702067822208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.234912:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067822208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.234919:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.234922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.234925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067822208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764392.234973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067822336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764392.234991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764392.235001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764392/real 1789764392] req@ffff9360061dc000 x1876702067822336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764392.235013:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067822336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764417 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764392.235021:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764392.235023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764392.235026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067822336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764393.005438:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764394.026786:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764395.050984:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764396.077303:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764397.098246:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764397.353983:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.353994:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764397.353999:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354011:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354026:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354036:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764397.354191:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067822464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354234:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.354238:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764397.354240:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354244:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354248:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354252:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354270:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764397.354282:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067822592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354292:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.354296:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764397.354298:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354301:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354304:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354308:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354312:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764397.354323:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067822720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354332:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.354335:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764397.354336:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354340:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354344:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354347:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354352:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764397.354361:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067822848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354369:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789764397.354369:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067822464:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764397.354372:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764397.354374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354376:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354379:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354383:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354386:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764397.354393:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067822976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354400:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.354403:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764397.354405:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354408:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354411:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354414:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354418:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764397.354425:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067823104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764397.354437:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764397.354439:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764397.354441:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764397.354444:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1865 00000100:00080000:2.0:1789764397.354447:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764397.354450:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764397.354454:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764397.354463:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067823232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764397.354535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764397.354565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764397.354597:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764397.354621:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764397.354642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067823104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764397.354660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067823232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764398.122329:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764399.149387:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764400.170029:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764400.426186:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764400.426241:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764400.426262:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764400.426310:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764400.426321:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764400.426333:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764400.426346:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764400.426356:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764400.426361:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061de680 x1876702067822848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789764400.426367:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764400.426396:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067822848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426411:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426417:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764400.426430:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061dfb80 x1876702067822976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067822976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426449:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426453:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764400.426463:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061df800 x1876702067823104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067823104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067823104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764400.426496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061dca80 x1876702067823232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067823232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426523:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067823232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764400.426529:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061dd180 x1876702067822464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426540:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067822464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426548:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426550:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067822464:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764400.426560:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061dce00 x1876702067822592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067822592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426580:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426583:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426587:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764400.426595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764397/real 1789764400] req@ffff9360061dc000 x1876702067822720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764400.426607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067822720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764422 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764400.426615:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764400.426618:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764400.426621:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067822720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764401.194039:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764402.219309:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764402.474193:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764402.474208:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474219:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474233:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474243:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764402.474302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067823360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764402.474346:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474350:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474354:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474358:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764402.474384:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067823488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764402.474399:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474404:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474410:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474415:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764402.474425:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067823616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474434:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474437:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764402.474439:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474443:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474446:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474450:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474454:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764402.474464:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067823744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474473:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474476:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764402.474478:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474482:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474486:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474489:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474494:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764402.474510:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067823872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764402.474525:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474528:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474531:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474535:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474539:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764402.474549:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067824000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764402.474557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764402.474561:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764402.474562:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764402.474566:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1870 00000100:00080000:0.0:1789764402.474570:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764402.474573:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764402.474577:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764402.474587:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067824128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764402.474923:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067823360:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764402.475019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475071:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020dac380 x1876702067823360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067823360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475102:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067823360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764402.475120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764402.475144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475156:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020daf100 x1876702067823488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475167:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067823488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475175:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475178:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764402.475187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764402.475209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475220:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020dac700 x1876702067823616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475231:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067823616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475240:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764402.475253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764402.475380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020dafb80 x1876702067823744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067823744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475410:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475413:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475416:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764402.475422:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764402.475441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475451:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020dac000 x1876702067823872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475462:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067823872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067823872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764402.475479:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764402.475497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475506:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020dae680 x1876702067824000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475523:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067824000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764402.475543:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764402.475562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764402.475574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764402/real 1789764402] req@ffff936020daf480 x1876702067824128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764402.475586:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067824128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764402.475593:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764402.475596:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764402.475600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764403.244286:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764404.267874:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764405.290418:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764406.315927:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764407.338042:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764407.593986:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.593996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764407.593999:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594010:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594020:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594028:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764407.594067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067824256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594103:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764407.594108:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594116:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594119:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594128:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764407.594143:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067824384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764407.594161:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594165:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594170:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594175:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594185:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764407.594198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067824512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594209:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764407.594211:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594215:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594218:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594221:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594225:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764407.594233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067824640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764407.594244:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594247:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594249:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594252:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594256:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764407.594264:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067824768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764407.594274:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594277:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594280:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594283:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594287:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764407.594294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067824896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764407.594301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764407.594304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764407.594305:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764407.594308:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1875 00000100:00080000:2.0:1789764407.594310:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764407.594313:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764407.594317:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764407.594324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067825024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764407.594518:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067824256:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764407.594580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594634:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061dd500 x1876702067824256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.594652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067824256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.594662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.594666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.594670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067824256:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764407.594677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764407.594699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061df480 x1876702067824384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.594723:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067824384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.594745:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.594751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.594755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764407.594761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764407.594794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594842:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061df100 x1876702067824512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.594856:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067824512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.594868:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.594871:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.594874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764407.594879:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764407.594902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594913:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061dd880 x1876702067824640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.594924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067824640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.594932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.594935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.594937:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764407.594943:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764407.594958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594964:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061ddc00 x1876702067824768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.594971:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067824768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.594976:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.594977:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.594980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067824768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764407.594983:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764407.594993:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.594998:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061de300 x1876702067824896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.595004:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067824896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.595009:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.595010:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.595011:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067824896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764407.595014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764407.595023:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764407.595028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764407/real 1789764407] req@ffff9360061ddf80 x1876702067825024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764407.595033:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067825024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764407.595037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764407.595039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764407.595041:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764408.361992:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764409.385987:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764410.409963:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764411.433992:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764412.457990:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764412.713930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.713938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764412.713942:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.713949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.713957:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.713962:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764412.713991:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067825152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764412.714014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764412.714017:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714020:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714022:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714023:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714031:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764412.714038:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067825280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764412.714046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764412.714049:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714053:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714055:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714058:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714061:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764412.714069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067825408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764412.714071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067825152:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764412.714094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764412.714097:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714100:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714103:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714106:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764412.714114:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067825536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764412.714120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764412.714123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714125:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714127:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714129:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714131:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764412.714138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067825664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764412.714143:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714146:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764412.714147:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714149:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714152:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714155:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714164:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789764412.714164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764412.714177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067825792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764412.714184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764412.714186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764412.714187:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764412.714191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1880 00000100:00080000:0.0:1789764412.714197:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764412.714199:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764412.714201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764412.714207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067825920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764412.714234:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020daed80 x1876702067825152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714252:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067825152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714269:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714273:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067825152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764412.714282:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764412.714308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020dad880 x1876702067825280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067825280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714332:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714334:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764412.714358:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764412.714373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020dad500 x1876702067825408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714386:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067825408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714391:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714393:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764412.714399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764412.714410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020daf480 x1876702067825536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067825536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714429:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714432:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764412.714440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764412.714458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714469:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020dae680 x1876702067825664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067825664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714488:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714490:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067825664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764412.714496:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764412.714512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020dac000 x1876702067825792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067825792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714547:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764412.714552:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764412.714571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764412.714581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764412/real 1789764412] req@ffff936020dafb80 x1876702067825920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764412.714593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067825920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764412.714606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764412.714609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764412.714612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067825920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764413.482612:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764414.507197:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764415.530167:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764416.555203:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764417.578048:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764417.833966:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.833977:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764417.833982:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.833995:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834008:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834017:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764417.834061:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067826048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764417.834100:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.834110:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764417.834112:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834116:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834120:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834124:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834135:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764417.834150:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067826176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764417.834160:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.834164:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764417.834174:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834179:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834183:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834186:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834191:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764417.834203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067826304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764417.834212:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.834215:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764417.834217:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834220:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834224:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834228:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834234:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764417.834244:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067826432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764417.834438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067826048:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764417.834486:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.834489:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764417.834492:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834496:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834504:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834509:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834516:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789764417.834518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764417.834532:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067826560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764417.834543:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764417.834546:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764417.834548:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834552:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834556:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834560:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834566:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764417.834576:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067826688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764417.834586:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789764417.834588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef7100 x1876702067826048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764417.834592:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764417.834596:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764417.834606:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1885 00000100:00080000:1.0:1789764417.834610:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764417.834614:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764417.834619:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764417.834622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067826048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.834635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.834639:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.834644:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067826048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764417.834651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764417.834677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764417.834688:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067826816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764417.834693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef4e00 x1876702067826176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.834730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067826176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.834740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.834744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.834748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764417.834755:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764417.834777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764417.834788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef6d80 x1876702067826304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.834839:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067826304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.834848:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.834852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.834855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764417.834892:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764417.834914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764417.834938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef6300 x1876702067826432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.834952:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067826432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.834959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.834963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.834967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764417.834974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764417.834994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764417.835005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef7b80 x1876702067826560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.835015:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067826560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.835023:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.835025:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.835028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067826560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764417.835034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067826688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764417.835052:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764417.835062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef7480 x1876702067826688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.835073:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067826688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.835080:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.835083:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.835086:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067826688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764417.835090:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067826816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764417.835108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764417.835119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764417/real 1789764417] req@ffff936010ef7800 x1876702067826816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764417.835129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067826816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764417.835136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764417.835139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764417.835141:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067826816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764418.602121:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764419.625938:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764420.650105:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764421.674178:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764422.697970:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764422.953936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.953948:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764422.953953:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.953964:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.953977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.953985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764422.954030:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067826944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764422.954061:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.954066:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764422.954067:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.954072:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954075:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954079:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764422.954091:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764422.954111:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067827072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764422.954117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067826944:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764422.954128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.954132:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764422.954134:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.954139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954143:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954147:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764422.954152:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764422.954167:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067827200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764422.954177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.954181:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764422.954183:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.954186:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954190:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954196:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:3.0:1789764422.954196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789764422.954204:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764422.954217:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067827328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764422.954226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.954229:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764422.954232:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.954236:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954240:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954244:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954248:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061dc380 x1876702067826944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789764422.954260:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764422.954271:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067826944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764422.954279:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067827456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764422.954286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764422.954293:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764422.954293:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789764422.954300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789764422.954301:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067826944:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789764422.954309:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764422.954315:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954319:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954323:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764422.954329:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764422.954330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764422.954346:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067827584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764422.954357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764422.954363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764422.954368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764422.954370:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789764422.954373:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061dc700 x1876702067827072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764422.954377:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1890 00000100:00080000:2.0:1789764422.954384:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764422.954391:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954396:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067827072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789764422.954400:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789764422.954410:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954414:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764422.954419:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067827712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764422.954421:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764422.954432:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764422.954454:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954466:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061ddf80 x1876702067827200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764422.954477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067827200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764422.954485:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954488:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764422.954491:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764422.954497:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764422.954513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061de300 x1876702067827328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764422.954533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067827328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764422.954540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764422.954546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764422.954551:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764422.954575:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954587:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061ddc00 x1876702067827456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764422.954597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067827456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764422.954605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764422.954611:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764422.954630:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067827584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764422.954646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061dd880 x1876702067827584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764422.954684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067827584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764422.954693:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954695:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764422.954700:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067827584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764422.954705:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067827712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764422.954722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764422.954731:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764422/real 1789764422] req@ffff9360061df100 x1876702067827712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764422.954741:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067827712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764422.954749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764422.954751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764422.954755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067827712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764423.722712:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764424.746006:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764425.770030:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764426.794052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764427.817976:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764428.073951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.073964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764428.073969:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764428.073978:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.073989:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.073995:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764428.074031:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067827840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764428.074068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.074071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764428.074072:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764428.074076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.074078:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074081:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764428.074088:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764428.074098:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067827968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764428.074104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.074107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764428.074108:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764428.074111:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.074113:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074117:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067827840:192.168.203.137@tcp:250:kworker.0 00000008:00100000:0.0:1789764428.074120:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764428.074131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067828096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764428.074140:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.074144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764428.074146:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764428.074150:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.074154:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074158:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764428.074163:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789764428.074174:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764428.074176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067828224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764428.074184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.074187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764428.074189:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764428.074193:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.074197:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074200:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764428.074205:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764428.074210:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020dac700 x1876702067827840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764428.074219:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067828352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764428.074226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067827840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764428.074235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764428.074238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764428.074239:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764428.074240:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764428.074244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764428.074247:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00100000:3.0:1789764428.074251:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067827840:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764428.074253:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074261:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827968:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789764428.074270:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789764428.074283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764428.074287:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067828480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764428.074295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020daf100 x1876702067827968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074314:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067827968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764428.074319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764428.074323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764428.074324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764428.074328:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764428.074329:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789764428.074332:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067827968:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764428.074337:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1895 00000100:00080000:0.0:1789764428.074342:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764428.074347:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764428.074351:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764428.074363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764428.074365:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067828608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764428.074386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074394:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020dac380 x1876702067828096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067828096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764428.074407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764428.074409:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764428.074411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764428.074414:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764428.074424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074430:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020dad180 x1876702067828224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067828224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764428.074441:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764428.074442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764428.074444:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764428.074447:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764428.074455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074460:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020dae300 x1876702067828352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067828352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764428.074471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764428.074473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764428.074474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764428.074477:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067828480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764428.074486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074491:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020daf800 x1876702067828480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067828480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764428.074502:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764428.074503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764428.074505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067828480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764428.074514:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067828608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764428.074523:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764428.074529:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764428/real 1789764428] req@ffff936020dadc00 x1876702067828608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764428.074535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067828608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764428.074539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764428.074541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764428.074543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067828608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764428.842906:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764429.866164:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764430.891262:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764431.914233:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764432.938510:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764433.196014:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196026:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764433.196030:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196040:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196050:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196057:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764433.196092:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067828736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.196159:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067828736:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764433.196218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764433.196247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764433.196252:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196255:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196259:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196262:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.196272:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764433.196285:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067828864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.196337:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061df480 x1876702067828736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.196352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067828736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.196362:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.196366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.196370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067828736:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764433.196393:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764433.196412:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764433.196423:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061dd500 x1876702067828864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.196434:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067828864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.196440:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.196443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.196445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828864:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764433.196525:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196530:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764433.196531:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196535:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196539:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196543:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.196547:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764433.196559:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067828992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.196606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764433.196625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764433.196645:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196649:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764433.196650:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196654:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196657:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196660:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.196665:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764433.196674:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067829120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.196703:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061dc000 x1876702067828992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.196713:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067828992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.196721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.196723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.196726:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067828992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764433.196736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764433.196753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764433.196762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061dce00 x1876702067829120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.196771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067829120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.196778:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.196781:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.196784:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829120:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789764433.196873:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196876:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764433.196878:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196882:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196885:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196888:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.196893:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764433.196903:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067829248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.196938:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764433.196957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789764433.196975:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.196978:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764433.196980:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.196983:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.196986:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.196990:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.196994:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764433.197004:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067829376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.197036:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061dd180 x1876702067829248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.197047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067829248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.197054:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.197056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.197059:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764433.197070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067829376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764433.197086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764433.197096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061dca80 x1876702067829376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.197107:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067829376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.197114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.197116:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.197118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067829376:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789764433.197188:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764433.197191:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764433.197193:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764433.197197:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1901 00000100:00080000:2.0:1789764433.197201:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764433.197204:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764433.197209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764433.197219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067829504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764433.197256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067829504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764433.197274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764433.197328:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764433/real 1789764433] req@ffff9360061df800 x1876702067829504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764433.197339:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067829504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764433.197347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764433.197350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764433.197353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067829504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764433.962903:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764434.986100:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764436.011532:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764437.035243:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764438.061491:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764438.313950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.313962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764438.313967:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.313980:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.313993:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314003:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764438.314050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067829632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764438.314092:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314097:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314101:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314105:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314116:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764438.314129:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067829760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764438.314145:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314149:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314152:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314156:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314160:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764438.314174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067829888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764438.314188:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314192:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314195:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314199:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314204:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764438.314214:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067830016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764438.314227:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314231:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314234:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314237:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314242:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764438.314265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067830144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314274:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764438.314280:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314283:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314286:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314290:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764438.314314:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067830272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764438.314329:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764438.314332:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764438.314334:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764438.314338:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1906 00000100:00080000:2.0:1789764438.314341:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764438.314344:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764438.314348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764438.314358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067830400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764438.314579:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067829632:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764438.314663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.314732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061dfb80 x1876702067829632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.314764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067829632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.314777:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.314782:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.314786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067829632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764438.314794:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764438.314848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.314862:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061de680 x1876702067829760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.314875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067829760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.314883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.314886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.314890:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764438.314896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764438.314916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.314928:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061df800 x1876702067829888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.314940:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067829888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.314953:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.314956:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.314959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067829888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764438.314965:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764438.314986:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.314996:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061dca80 x1876702067830016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.315006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067830016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.315014:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.315017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.315020:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764438.315025:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764438.315044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.315055:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061dd180 x1876702067830144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.315066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067830144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.315073:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.315076:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.315079:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764438.315085:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067830272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764438.315105:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.315115:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061dce00 x1876702067830272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.315125:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067830272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.315133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.315136:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.315139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067830272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764438.315144:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067830400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764438.315163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764438.315173:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764438/real 1789764438] req@ffff9360061dc000 x1876702067830400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764438.315184:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067830400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764463 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764438.315191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764438.315194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764438.315197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067830400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764439.082171:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764440.106061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764441.130001:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764442.155831:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764443.178149:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764443.433960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.433971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764443.433976:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.433987:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.433999:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434007:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764443.434054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067830528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764443.434093:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434097:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434100:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434103:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764443.434125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067830656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764443.434139:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434142:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434152:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764443.434160:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067830784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764443.434174:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434177:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434179:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434186:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764443.434193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067830912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764443.434204:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434207:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434210:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434213:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764443.434224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067831040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764443.434236:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434245:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434249:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764443.434258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067831168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764443.434265:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764443.434267:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764443.434269:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764443.434272:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1911 00000100:00080000:2.0:1789764443.434274:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764443.434277:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764443.434281:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764443.434288:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067831296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764443.435474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067830528:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764443.435571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.435658:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061dd500 x1876702067830528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.435679:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067830528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.435690:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.435694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.435699:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067830528:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764443.435707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764443.435737:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.435748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061df480 x1876702067830656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.435760:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067830656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.435769:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.435772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.435775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764443.435789:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764443.435842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.435852:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061df100 x1876702067830784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.435862:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067830784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.435870:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.435873:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.435876:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764443.435882:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764443.435898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.435906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061dd880 x1876702067830912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.435916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067830912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.435923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.435927:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.435930:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067830912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764443.435935:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764443.435949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.435957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061ddc00 x1876702067831040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.435976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067831040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.435983:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.435986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.435989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764443.435994:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067831168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764443.436009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.436017:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061de300 x1876702067831168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.436027:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067831168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.436034:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.436037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.436040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067831168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764443.436047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067831296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764443.436062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764443.436070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764443/real 1789764443] req@ffff9360061ddf80 x1876702067831296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764443.436080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067831296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764443.436093:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764443.436096:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764443.436099:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067831296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764444.202293:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764445.226753:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764446.250093:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764447.275944:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764448.299781:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764448.554764:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.554781:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764448.554785:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.554838:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.554849:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.554856:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764448.554904:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067831424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764448.554999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067831424:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764448.555073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764448.555109:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.555118:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764448.555121:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.555125:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.555130:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.555143:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.555158:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764448.555172:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067831552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764448.555752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa7b80 x1876702067831424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.555771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067831424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.555782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.555786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.555790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067831424:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764448.555859:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764448.555886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.555900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa7800 x1876702067831552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.555913:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067831552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.555921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.555924:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.555927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831552:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764448.557346:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.557354:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764448.557358:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.557365:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.557374:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.557380:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.557390:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764448.557416:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067831680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764448.557428:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.557431:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764448.557433:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.557436:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.557439:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.557443:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.557447:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764448.557457:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067831808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764448.557464:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.557467:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764448.557469:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.557482:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.557486:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.557490:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.557494:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764448.557503:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067831936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764448.557511:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.557514:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764448.557516:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.557519:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.557522:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.557525:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.557529:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764448.557536:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067832064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764448.557544:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764448.557546:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764448.557548:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764448.557550:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1916 00000100:00080000:3.0:1789764448.557553:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764448.557556:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764448.557560:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764448.557567:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067832192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764448.558049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764448.558097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.558125:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa6300 x1876702067831680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.558154:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067831680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.558163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.558167:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.558170:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764448.558177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764448.558351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.559320:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa7800 x1876702067831808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.559338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067831808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.559354:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.559358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.559361:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764448.559370:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764448.559402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.559415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa7b80 x1876702067831936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.559426:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067831936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.559434:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.559437:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.559440:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067831936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764448.559446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764448.559467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.559492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa5180 x1876702067832064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.559502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067832064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.559509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.559512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.559515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764448.559521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764448.559538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764448.559548:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764448/real 1789764448] req@ffff936010aa4380 x1876702067832192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764448.559558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067832192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764448.559565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764448.559568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764448.559571:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764449.322389:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764450.350832:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764451.370226:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764452.396205:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764453.418038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764453.674693:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.674706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764453.674711:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.674724:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.674738:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.674747:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764453.674834:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067832320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764453.675497:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067832320:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764453.675580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764453.676828:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.676837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764453.676840:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.676849:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.676858:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.676866:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.676891:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764453.676927:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067832448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764453.677154:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020dace00 x1876702067832320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.677179:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067832320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.677190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.677195:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.677199:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067832320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764453.677237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832448:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764453.677282:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.677286:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764453.677288:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.677292:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.677297:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.677301:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.677306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764453.677319:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067832576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764453.677328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.677331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764453.677333:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.677337:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.677340:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.677343:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.677348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764453.677360:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067832704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764453.677367:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.677371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764453.677372:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.677375:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.677378:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.677382:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.677387:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764453.677397:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067832832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764453.677404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.677408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764453.677409:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.677412:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.677415:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.677418:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.677422:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764453.677432:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067832960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764453.677591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.677610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020daea00 x1876702067832448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.677621:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067832448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.677628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.677631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.677636:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832448:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764453.677774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764453.677777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764453.677779:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764453.677784:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1921 00000100:00080000:0.0:1789764453.677789:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764453.677793:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764453.677824:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764453.677838:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067833088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764453.679015:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764453.679079:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.679102:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020dadf80 x1876702067832576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.679119:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067832576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.679129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.679133:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.679137:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764453.679144:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764453.679165:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.679176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020dace00 x1876702067832704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.679187:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067832704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.679194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.679198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.679201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764453.679208:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764453.679228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.679240:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020dadc00 x1876702067832832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.679252:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067832832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.679260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.679263:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.679266:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067832832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764453.679273:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764453.679293:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.679303:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020daf800 x1876702067832960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.679313:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067832960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.679320:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.679323:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.679326:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067832960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764453.679332:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764453.679350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764453.679361:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764453/real 1789764453] req@ffff936020dae300 x1876702067833088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764453.679371:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067833088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764453.679379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764453.679382:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764453.679385:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764454.442124:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764455.466433:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764456.490290:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764457.514157:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764458.538018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764458.793962:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.793973:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764458.793979:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.793992:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794006:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794016:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764458.794074:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067833216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794109:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794121:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764458.794123:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794127:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794131:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794136:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794146:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764458.794159:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067833344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794168:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794172:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764458.794173:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794178:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794181:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794185:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794189:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764458.794202:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067833472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794211:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794214:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764458.794216:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794219:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794222:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794226:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794230:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764458.794239:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067833600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764458.794252:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794255:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794258:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794261:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794266:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764458.794274:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067833728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794283:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794286:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764458.794288:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794291:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794294:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794299:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794304:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764458.794314:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067833856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764458.794323:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764458.794326:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764458.794328:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764458.794331:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1926 00000100:00080000:2.0:1789764458.794335:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764458.794338:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764458.794342:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764458.794351:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067833984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764458.794656:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067833216:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764458.794738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.794909:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061dc700 x1876702067833216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.794928:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067833216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.794940:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.794944:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.794949:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067833216:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764458.794956:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764458.794983:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.794997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061dc380 x1876702067833344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.795008:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067833344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.795017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.795222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.795227:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764458.795235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764458.795262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.795274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061ddf80 x1876702067833472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.795286:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067833472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.795294:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.795298:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.795301:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764458.795309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764458.795331:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.795344:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061de300 x1876702067833600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.795355:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067833600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.795363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.795366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.795370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764458.795376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764458.796072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.796088:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061ddc00 x1876702067833728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.796101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067833728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.796111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.796895:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.796908:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067833728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764458.796920:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764458.796964:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.796985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061dd880 x1876702067833856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.797001:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067833856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.797010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.797014:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.797017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764458.797024:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764458.797043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764458.797054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764458/real 1789764458] req@ffff9360061df100 x1876702067833984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764458.797066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067833984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764458.797074:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764458.797077:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764458.797081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067833984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764459.562087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764460.586035:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764461.610089:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764462.634136:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764463.663625:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764463.914009:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.914019:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764463.914024:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.914035:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.914048:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.914056:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764463.914100:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067834112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764463.914183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067834112:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764463.916673:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.916682:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764463.916686:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.916694:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.916703:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.916711:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.916726:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764463.916760:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067834240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764463.917626:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834240:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764463.919755:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.919763:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764463.919767:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.919775:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.919784:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.919790:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.919823:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764463.919853:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067834368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764463.919869:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.919872:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764463.919874:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.919878:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.919881:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.919885:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.919890:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764463.919900:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067834496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764463.919908:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.919912:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764463.919914:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.919917:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.919919:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.919923:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.919927:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764463.919935:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067834624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764463.919943:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.919947:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764463.919948:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.919952:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.919954:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.919958:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.919963:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764463.919971:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067834752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764463.920528:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764463.920647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764463.920671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764463.920693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067834752:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764463.920875:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764463.920880:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764463.920882:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764463.920887:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1931 00000100:00080000:3.0:1789764463.920891:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764463.920895:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764463.920902:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764463.920918:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067834880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764463.921482:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067834880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764464.682254:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764465.706500:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764466.731829:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789764466.993454:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:0.0:1789764466.993510:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789764466.993532:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993588:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993598:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993607:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993614:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993621:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764466.993629:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764466.993890:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa6a00 x1876702067834880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.993918:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067834880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.993931:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.993936:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.993940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067834880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764466.993948:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa4700 x1876702067834368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.993959:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067834368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.993967:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.993971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.993974:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764466.993981:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa7100 x1876702067834496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.993999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067834496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.994006:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.994009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.994013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764466.994019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa4000 x1876702067834624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.994029:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067834624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.994037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.994041:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.994044:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764466.994279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa5c00 x1876702067834752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.994296:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067834752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.994304:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.994308:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.994311:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067834752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764466.994318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa5880 x1876702067834240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.994329:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067834240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.994348:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.994352:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.994355:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067834240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764466.994362:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764463/real 1789764466] req@ffff936010aa4a80 x1876702067834112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764466.994375:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067834112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764466.994383:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764466.994385:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764466.994388:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067834112:192.168.203.137@tcp:250:kworker.0 00010000:00010000:3.0:1789764467.756457:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764468.778071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764469.033950:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.033962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764469.033968:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.033980:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.033995:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034004:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764469.034053:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067835008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764469.034102:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034110:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034125:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764469.034138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067835136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764469.034152:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034157:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034160:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034163:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764469.034177:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067835264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034185:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764469.034190:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034194:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034198:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034202:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034206:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764469.034216:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067835392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764469.034229:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034233:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034236:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034239:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764469.034251:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067835520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034262:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764469.034263:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034267:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034270:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034274:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764469.034288:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067835648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764469.034302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764469.034305:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764469.034307:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764469.034310:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1936 00000100:00080000:3.0:1789764469.034313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764469.034316:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764469.034331:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764469.034342:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067835776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764469.034455:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067835008:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764469.034555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa4a80 x1876702067835008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034628:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067835008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034639:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067835008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764469.034655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764469.034679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa5880 x1876702067835136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067835136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764469.034724:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764469.034744:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034755:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa5c00 x1876702067835264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034767:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067835264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034783:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764469.034791:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764469.034842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034853:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa4000 x1876702067835392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034863:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067835392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034871:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034874:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034878:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764469.034883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764469.034901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa7100 x1876702067835520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034922:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067835520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034929:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034932:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034936:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067835520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764469.034941:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067835648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764469.034960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.034970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa4700 x1876702067835648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.034981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067835648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.034989:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.034992:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.034995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067835648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764469.035000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067835776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764469.035018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764469.035029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764469/real 1789764469] req@ffff936010aa6a00 x1876702067835776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764469.035040:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067835776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764469.035047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764469.035050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764469.035052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067835776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764469.806037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764470.826127:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764471.851996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764472.874675:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764473.901852:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764474.154555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.154565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764474.154571:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.154583:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:0.0:1789764474.154599:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764474.154608:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764474.154657:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067835904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764474.154696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.154701:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764474.154703:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.154707:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:0.0:1789764474.154710:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764474.154714:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764474.154724:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764474.154735:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067836032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764474.154743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.154746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764474.154748:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.154751:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:0.0:1789764474.154754:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764474.154757:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764474.154761:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764474.154770:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067836160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764474.154782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.154786:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764474.154788:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.154791:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:0.0:1789764474.154794:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789764474.154853:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067835904:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764474.154938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.154991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dad180 x1876702067835904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067835904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764474.155021:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155025:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764474.155029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067835904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764474.155038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764474.155063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.155076:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dac380 x1876702067836032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067836032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764474.155099:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:3.0:1789764474.155099:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000008:00100000:0.0:1789764474.155110:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764474.155115:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764474.155124:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764474.155128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067836288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764474.155141:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.155145:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764474.155147:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.155151:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1941 00000800:00000100:3.0:1789764474.155153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764474.155158:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764474.155163:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764474.155168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764474.155169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020daf100 x1876702067836160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764474.155181:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067836416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764474.155186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067836160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764474.155200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.155206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764474.155207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764474.155208:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.155217:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:3.0:1789764474.155217:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764474.155224:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789764474.155226:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836160:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764474.155230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764474.155236:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764474.155246:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067836544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764474.155254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764474.155258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764474.155260:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764474.155264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1941 00000100:00080000:0.0:1789764474.155268:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789764474.155269:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836288:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764474.155273:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764474.155279:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764474.155305:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067836672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764474.155313:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.155332:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dac700 x1876702067836288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155348:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067836288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764474.155357:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155361:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764474.155364:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764474.155369:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764474.155388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.155399:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dafb80 x1876702067836416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067836416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764474.155416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764474.155422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764474.155427:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067836544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764474.155446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.155457:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dac000 x1876702067836544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155468:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067836544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764474.155481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155485:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764474.155488:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067836544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764474.155503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067836672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764474.155710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764474.155790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764474/real 1789764474] req@ffff936020dae680 x1876702067836672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764474.155834:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067836672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764474.155842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764474.155845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764474.155847:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067836672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764474.922365:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764475.946766:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764476.969987:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764477.995572:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764479.022754:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764479.274727:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274734:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764479.274737:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274747:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274756:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274762:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764479.274793:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067836800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274833:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764479.274835:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274838:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274841:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274845:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.274853:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764479.274863:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067836928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274871:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764479.274875:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274878:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274881:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274884:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.274889:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764479.274896:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067837056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274903:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274906:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764479.274907:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274911:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274914:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274917:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.274920:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764479.274927:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067837184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274938:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764479.274939:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274942:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274944:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274946:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.274949:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764479.274958:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067837312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764479.274969:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.274972:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.274974:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.274977:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.274981:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764479.274988:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067837440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764479.274996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764479.274998:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764479.275000:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764479.275003:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1947 00000100:00080000:3.0:1789764479.275006:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764479.275010:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764479.275013:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764479.275021:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067837568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764479.275115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067836800:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764479.275337:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.275590:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa7480 x1876702067836800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.275610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067836800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.275620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.275624:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.275629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067836800:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764479.275637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764479.275666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.275685:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa4e00 x1876702067836928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.275697:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067836928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.275704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.275706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.275709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067836928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764479.275715:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764479.275731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.275740:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa6680 x1876702067837056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.275750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067837056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.275756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.275759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.275762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764479.275767:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764479.275782:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.275791:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa5f80 x1876702067837184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.275832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067837184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.275840:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.275842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.275844:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764479.275850:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764479.275866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.275876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa5500 x1876702067837312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.275887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067837312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.275968:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.275972:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.275975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764479.275981:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067837440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764479.276001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.276012:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa6a00 x1876702067837440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.276023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067837440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.276030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.276033:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.276036:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067837440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764479.276041:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067837568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764479.276061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764479.276071:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764479/real 1789764479] req@ffff936010aa4700 x1876702067837568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764479.276080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067837568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764504 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764479.276087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764479.276090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764479.276092:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067837568:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764480.042044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764481.080207:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764482.090180:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764483.115300:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764484.141106:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764484.395161:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395173:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764484.395179:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395201:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395223:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395232:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764484.395284:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067837696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395317:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395322:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764484.395324:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395328:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395332:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395337:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395348:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764484.395360:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067837824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395369:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395373:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764484.395374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395378:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395382:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395386:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395390:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764484.395401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067837952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395418:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395422:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764484.395424:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395428:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395431:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395435:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395440:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764484.395450:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067838080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395458:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395461:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764484.395463:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395466:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395469:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395473:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764484.395487:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067838208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395504:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395507:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764484.395509:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395513:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395516:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395529:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395535:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764484.395547:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067838336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764484.395555:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764484.395558:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764484.395560:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764484.395564:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1952 00000100:00080000:2.0:1789764484.395568:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764484.395571:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764484.395576:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764484.395585:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067838464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764484.395876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067837696:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764484.395955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396074:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061df480 x1876702067837696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067837696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396104:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067837696:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764484.396119:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764484.396144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396156:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061dd500 x1876702067837824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396167:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067837824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396174:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396177:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764484.396186:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764484.396226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061dc000 x1876702067837952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396249:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067837952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396264:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067837952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764484.396269:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764484.396291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396301:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061dce00 x1876702067838080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396312:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067838080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396318:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396321:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764484.396329:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764484.396346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061dd180 x1876702067838208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396365:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067838208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396372:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396375:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764484.396383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067838336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764484.396401:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061dca80 x1876702067838336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067838336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067838336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764484.396657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067838464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764484.396679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764484.396690:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764484/real 1789764484] req@ffff9360061df800 x1876702067838464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764484.396700:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067838464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764509 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764484.396706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764484.396709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764484.396712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067838464:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764485.163576:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764486.187089:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764487.210380:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764488.234071:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764489.259737:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764489.516844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.516857:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764489.516861:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.516872:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.516883:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.516890:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764489.516928:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067838592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.516957:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.516961:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764489.516963:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.516966:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.516969:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.516972:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.516981:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764489.516993:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067838720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.517001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.517004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764489.517005:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.517018:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.517022:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.517025:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.517029:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764489.517039:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067838848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.517047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.517050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764489.517051:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.517055:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.517058:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.517062:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.517066:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764489.517075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067838976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.517083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.517086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764489.517088:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.517091:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.517094:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.517097:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.517101:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764489.517249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067838592:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764489.517321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.517359:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020daf480 x1876702067838592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.517374:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067838592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.517394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.517397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.517400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067838592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764489.517407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764489.517429:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.517440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020dad500 x1876702067838720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.517451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067838720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.517457:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.517460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.517462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764489.517467:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764489.517485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.517494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020dad880 x1876702067838848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.517504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067838848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.517517:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.517519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.517522:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764489.517527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764489.517545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.517556:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020daed80 x1876702067838976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.517566:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067838976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.517574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.517577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.517580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067838976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764489.517770:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067839104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.517790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.517793:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764489.517795:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.517816:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.517821:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.517825:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.517831:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764489.517841:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067839232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764489.517849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764489.517851:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764489.517854:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764489.517857:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1957 00000100:00080000:0.0:1789764489.517860:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764489.517863:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764489.517867:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764489.517875:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067839360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764489.518295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764489.518320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.518331:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020dae680 x1876702067839104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.518342:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067839104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.518350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.518353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.518357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764489.518363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067839232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764489.518501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.518514:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020dac000 x1876702067839232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.518526:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067839232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.518534:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.518537:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.518541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067839232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764489.518546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067839360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764489.518568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764489.518578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764489/real 1789764489] req@ffff936020dafb80 x1876702067839360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764489.518588:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067839360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764489.518595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764489.518598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764489.518600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067839360:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764490.282047:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764491.306245:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764492.330112:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764493.357581:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764494.379198:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764494.634394:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634406:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764494.634409:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634420:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634431:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634439:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764494.634492:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067839488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634525:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634528:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764494.634530:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634534:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634537:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634541:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764494.634549:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764494.634560:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067839616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634570:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634573:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764494.634575:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634579:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634583:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634586:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764494.634590:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764494.634601:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067839744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634610:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634614:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789764494.634617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067839488:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764494.634617:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634625:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634629:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634632:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764494.634637:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764494.634649:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067839872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634658:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634662:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764494.634663:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634666:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634670:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634674:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764494.634678:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764494.634688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764494.634689:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067840000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634700:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634704:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764494.634706:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764494.634710:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634713:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764494.634717:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764494.634722:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764494.634733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa7100 x1876702067839488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764494.634734:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067840128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764494.634763:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764494.634766:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764494.634768:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764494.634772:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067839488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764494.634775:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1962 00000100:00080000:3.0:1789764494.634782:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764494.634789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764494.634789:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764494.634822:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.634827:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067839488:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764494.634836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839616:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789764494.634839:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764494.634856:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067840256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764494.634867:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.634883:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa4000 x1876702067839616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.634896:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067839616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.634909:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.634912:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.634915:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764494.634942:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764494.634961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.634972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa5c00 x1876702067839744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.634982:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067839744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.634989:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.634992:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.634996:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764494.635001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764494.635033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.635045:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa5880 x1876702067839872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.635056:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067839872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.635063:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.635066:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.635079:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067839872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764494.635086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764494.635106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.635116:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa4a80 x1876702067840000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.635126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067840000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.635234:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.635239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.635242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764494.635249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067840128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764494.635339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.635352:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa4380 x1876702067840128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.635364:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067840128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.635371:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.635374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.635376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067840128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764494.635382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067840256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764494.635399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764494.635409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764494/real 1789764494] req@ffff936010aa5180 x1876702067840256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764494.635419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067840256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764494.635426:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764494.635428:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764494.635431:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067840256:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764495.402180:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764496.426102:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764497.451091:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764498.475165:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764499.498021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764499.761569:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761582:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764499.761587:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761598:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761608:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761616:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764499.761656:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067840384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761693:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761697:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764499.761699:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761703:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761707:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761710:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761719:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764499.761731:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067840512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761742:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761745:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764499.761747:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761751:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761754:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761757:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761761:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764499.761773:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067840640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761781:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761784:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764499.761786:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761788:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761791:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761795:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761836:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764499.761847:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067840768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761855:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761858:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764499.761860:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761863:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761867:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761870:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761874:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764499.761883:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067840896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761891:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761894:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764499.761895:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761899:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761901:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761905:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761909:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764499.761918:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067841024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764499.761940:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764499.761942:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764499.761945:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764499.761948:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1967 00000100:00080000:2.0:1789764499.761952:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764499.761956:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764499.761961:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764499.761971:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067841152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764499.762254:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067840384:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764499.762315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061de680 x1876702067840384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762383:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067840384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762393:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067840384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764499.762409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764499.762432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061dfb80 x1876702067840512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067840512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762463:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764499.762473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764499.762489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762498:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061df800 x1876702067840640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067840640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764499.762531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764499.762549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061dca80 x1876702067840768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762569:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067840768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762580:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762583:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764499.762588:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764499.762606:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061dd180 x1876702067840896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067840896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067840896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764499.762644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764499.762661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762670:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061dce00 x1876702067841024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067841024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762687:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764499.762699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764499.762717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764499.762727:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764499/real 1789764499] req@ffff9360061dc000 x1876702067841152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764499.762738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067841152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764499.762746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764499.762749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764499.762752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841152:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764500.522415:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764501.546759:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764502.570452:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764503.594007:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764504.620506:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764504.873945:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.873958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764504.873963:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.873975:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.874186:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.874195:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764504.874438:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067841280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764504.874481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.874485:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764504.874486:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.874490:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.874495:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.874499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764504.875270:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067841280:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764504.875332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764504.875372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa7b80 x1876702067841280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764504.875390:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067841280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764504.875400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764504.875403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764504.875406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067841280:192.168.203.137@tcp:250:kworker.0 00000008:00100000:3.0:1789764504.875926:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764504.875948:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067841408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764504.875967:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.875971:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764504.875973:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.875978:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.875983:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.875989:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764504.875994:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764504.876004:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067841536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764504.876013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.876016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764504.876018:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.876021:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.876024:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.876027:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764504.876031:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764504.876040:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067841664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764504.876047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.876049:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764504.876051:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.876054:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.876056:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.876073:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764504.876078:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764504.876088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067841792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764504.876230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764504.876264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.876279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa7800 x1876702067841408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.876293:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067841408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.876301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.876305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.876308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764504.876313:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764504.876332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.876343:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa6300 x1876702067841536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.876354:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067841536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.876363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.876366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.876369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764504.876375:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764504.876393:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.876403:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa7b80 x1876702067841664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.876413:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067841664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.876421:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.876424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.876427:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841664:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764504.877191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.877196:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764504.877199:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.877256:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.877262:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.877268:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764504.877276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764504.877294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067841920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764504.877303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764504.877306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764504.877307:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764504.877310:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1972 00000100:00080000:3.0:1789764504.877313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764504.877317:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764504.877321:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764504.877330:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067842048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764504.877507:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764504.877615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.877634:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa5180 x1876702067841792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.877649:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067841792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.877658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.877661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.877665:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067841792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764504.877672:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764504.877691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.877703:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa4380 x1876702067841920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.877715:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067841920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.877723:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.877726:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.877729:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067841920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764504.877735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764504.877753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764504.877764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764504/real 1789764504] req@ffff936010aa4a80 x1876702067842048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764504.877775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067842048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764504.877783:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764504.877786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764504.877789:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842048:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764505.642358:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764506.666530:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764507.691184:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764508.714908:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764509.737996:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764509.994407:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764509.994419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764509.994423:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764509.994433:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764509.994443:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764509.994450:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764509.994486:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067842176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764509.994556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067842176:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764509.994616:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764509.994649:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764509.994653:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764509.994655:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764509.994659:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764509.994664:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764509.994667:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764509.994676:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764509.994688:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067842304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764509.999882:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764509/real 1789764509] req@ffff936010ef5500 x1876702067842176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764509.999946:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067842176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764509.999958:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764509.999962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764509.999966:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067842176:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764509.999996:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764510.000062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764510.000078:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764509/real 1789764510] req@ffff936010ef6a00 x1876702067842304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764510.000089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067842304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.000097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.000100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.000103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842304:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764510.002901:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764510.002912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764510.002915:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764510.002925:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764510.002935:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764510.002942:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764510.002954:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764510.002984:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067842432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764510.003050:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764510.003103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764510.003130:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764510.003133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764510.003135:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764510.003139:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764510.003143:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764510.003291:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764510.003297:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764510.003310:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067842560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764510.003352:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764510/real 1789764510] req@ffff936010ef5f80 x1876702067842432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764510.004018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764510.004032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764510.004034:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764510.004039:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764510.004044:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764510.004049:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764510.004055:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764510.004069:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067842688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764510.004078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764510.004080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764510.004082:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764510.004085:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764510.004088:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764510.004091:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764510.004094:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764510.004102:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067842816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764510.004110:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764510.004112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764510.004114:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764510.004117:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1977 00000100:00080000:1.0:1789764510.004119:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764510.004122:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764510.004125:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764510.004131:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067842944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764510.004267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067842432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.004277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.004280:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.004284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764510.004305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764510.004332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764510.004345:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764510/real 1789764510] req@ffff936010ef6680 x1876702067842560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764510.004371:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067842560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.004379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.004382:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.004385:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764510.004390:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764510.004409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764510.004419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764510/real 1789764510] req@ffff936010ef6a00 x1876702067842688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764510.004428:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067842688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.004440:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.004443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.004445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067842688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764510.004451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764510.004467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764510.004476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764510/real 1789764510] req@ffff936010ef5500 x1876702067842816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764510.004486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067842816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.004493:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.004495:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.004498:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764510.004502:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764510.004516:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764510.004525:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764510/real 1789764510] req@ffff936010ef7800 x1876702067842944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764510.004533:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067842944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764510.004540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764510.004542:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764510.004545:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067842944:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764510.763194:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764511.787257:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764512.810050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764513.834058:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764514.863462:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764515.115569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.115581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764515.115586:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.115598:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.115613:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.115623:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764515.115672:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067843072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764515.116838:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067843072:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764515.116926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764515.118081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.118088:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764515.118093:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.118101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.118109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.118116:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.118128:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764515.118162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067843200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764515.118312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef7480 x1876702067843072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.118336:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067843072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.118348:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.118353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.118357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067843072:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764515.118387:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764515.118426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.118442:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef7b80 x1876702067843200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.118453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067843200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.118461:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.118464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.118467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843200:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764515.123513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.123523:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764515.123528:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.123540:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.123554:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.123563:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.123578:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764515.123624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067843328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764515.123642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.123646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764515.123648:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.123652:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.123656:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.123660:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.123665:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764515.123676:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067843456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764515.123685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.123688:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764515.123690:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.123694:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.123697:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.123700:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.123705:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764515.123715:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067843584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764515.123724:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.123728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764515.123730:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.123734:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.123738:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.123742:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.123747:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764515.123756:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067843712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764515.123764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764515.123767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764515.123768:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764515.123772:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1982 00000100:00080000:1.0:1789764515.123775:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764515.123778:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764515.123782:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764515.123792:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067843840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764515.123947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764515.124300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.124333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef6300 x1876702067843328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.124353:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067843328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.124364:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.124369:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.124373:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764515.124381:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764515.124405:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.124418:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef6d80 x1876702067843456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.124430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067843456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.124438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.124442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.124445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764515.124451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764515.124474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.124485:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef4e00 x1876702067843584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.124503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067843584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.124510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.124513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.124516:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067843584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764515.124522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067843712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764515.124543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.124555:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef7100 x1876702067843712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.124566:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067843712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.124573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.124577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.124580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067843712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764515.124586:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067843840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764515.124605:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764515.124616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764515/real 1789764515] req@ffff936010ef4380 x1876702067843840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764515.124627:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067843840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764515.124639:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764515.124642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764515.124645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067843840:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764515.884064:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764516.907577:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764517.930113:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764518.954416:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764519.978679:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764520.233998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764520.234014:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234026:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234040:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234049:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764520.234096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067843968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764520.234146:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234151:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234155:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234159:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234169:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764520.234182:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067844096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764520.234196:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234199:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234203:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234206:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234210:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764520.234221:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067844224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764520.234236:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234240:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234243:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234246:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234251:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764520.234260:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067844352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764520.234273:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234277:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234280:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234299:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764520.234312:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067844480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234318:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234321:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764520.234323:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234327:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234330:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234333:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234337:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764520.234348:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067844608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764520.234355:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764520.234358:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764520.234360:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764520.234573:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1988 00000100:00080000:3.0:1789764520.234579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764520.234584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764520.234589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764520.234603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067844736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764520.234833:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067843968:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764520.234925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.234991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa5880 x1876702067843968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067843968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235021:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235030:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067843968:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764520.235038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764520.235061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235074:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa5c00 x1876702067844096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235084:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067844096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235093:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235096:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235099:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764520.235106:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764520.235187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa4000 x1876702067844224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067844224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235221:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235224:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235227:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764520.235233:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764520.235254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa7100 x1876702067844352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235276:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067844352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764520.235561:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764520.235587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa4700 x1876702067844480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235612:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067844480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235619:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235626:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764520.235632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067844608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764520.235653:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235664:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa6a00 x1876702067844608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235676:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067844608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235683:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067844608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764520.235695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067844736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764520.235714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764520.235725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764520/real 1789764520] req@ffff936010aa5500 x1876702067844736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764520.235743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067844736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764545 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764520.235752:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764520.235754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764520.235758:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067844736:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764521.004714:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764522.026544:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764523.050026:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764524.074390:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764525.099647:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764525.354228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764525.354244:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354256:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354270:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354279:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764525.354329:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067844864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354368:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354372:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764525.354374:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354379:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354382:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354386:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354396:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764525.354408:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067844992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764525.354422:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354426:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354430:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354433:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354438:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764525.354448:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067845120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354460:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764525.354462:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354466:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354469:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354477:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764525.354491:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067845248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354503:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764525.354504:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354508:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354511:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354514:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354518:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764525.354529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067845376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354536:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764525.354540:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354543:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354546:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354550:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354554:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764525.354562:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067845504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764525.354570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764525.354573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764525.354574:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764525.354578:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1993 00000100:00080000:3.0:1789764525.354581:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764525.354583:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764525.354586:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764525.354593:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067845632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764525.354724:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067844864:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764525.354824:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.354875:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa5f80 x1876702067844864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.354894:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067844864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.354905:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.354920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.354924:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067844864:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764525.354932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764525.354958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.354971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa6680 x1876702067844992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.354981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067844992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.354990:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.354993:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.354996:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067844992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764525.355001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764525.355019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.355029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa4e00 x1876702067845120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.355039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067845120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.355046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.355049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.355052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764525.355057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764525.355077:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.355087:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa7480 x1876702067845248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.355097:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067845248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.355104:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.355106:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.355110:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764525.355115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764525.355133:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.355144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa5500 x1876702067845376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.355155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067845376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.355162:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.355165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.355168:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764525.355173:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067845504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764525.355192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.355203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa6a00 x1876702067845504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.355215:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067845504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.355223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.355226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.355229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067845504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764525.355235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067845632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764525.355252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764525.355264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764525/real 1789764525] req@ffff936010aa4700 x1876702067845632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764525.355275:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067845632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764550 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764525.355282:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764525.355284:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764525.355287:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067845632:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764526.122084:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764527.146134:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764528.170086:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764529.194257:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764530.218122:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764530.476361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764530.476378:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476391:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476406:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476414:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764530.476460:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067845760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764530.476499:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476504:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476508:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476511:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476531:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764530.476544:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067845888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764530.476559:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476563:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476566:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476574:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764530.476585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067846016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476597:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764530.476599:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476603:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476606:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476610:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476614:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764530.476625:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067846144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476633:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476637:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764530.476638:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476642:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476645:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476649:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476654:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764530.476664:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067846272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764530.476677:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476680:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476684:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476688:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476692:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764530.476704:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067846400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764530.476713:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764530.476716:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764530.476718:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764530.476722:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 1998 00000100:00080000:3.0:1789764530.476726:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764530.476730:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764530.476734:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764530.476743:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067846528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764530.477570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067845760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764530.477683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764530.477712:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764530.477732:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764530.477752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764530.477771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067846400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764530.477789:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067846528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764531.246527:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764532.268141:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764533.290042:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789764533.546252:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789764533.546299:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764533.549259:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549300:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549308:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549327:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549334:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549341:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764533.549349:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764533.549505:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa7100 x1876702067845760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067845760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549552:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067845760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764533.549559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa4000 x1876702067845888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067845888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549580:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549583:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067845888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764533.549590:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa5c00 x1876702067846016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549600:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067846016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549608:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549611:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549614:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764533.549621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa5880 x1876702067846144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067846144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549646:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764533.549656:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa4a80 x1876702067846272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067846272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764533.549688:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa4380 x1876702067846400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549697:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067846400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549703:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549707:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067846400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764533.549713:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764530/real 1789764533] req@ffff936010aa5180 x1876702067846528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764533.549722:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067846528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764533.549729:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764533.549731:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764533.549734:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067846528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764534.314055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764535.337983:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764535.593979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.593995:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764535.593998:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594010:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594020:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594027:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764535.594058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067846656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764535.594099:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594102:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594105:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594109:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594117:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764535.594128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067846784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594137:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594140:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764535.594141:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594145:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594148:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594151:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764535.594163:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067846912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764535.594381:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594385:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594389:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594393:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594398:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764535.594410:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067847040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594421:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764535.594423:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594426:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594429:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594432:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594436:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764535.594443:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067847168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764535.594457:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594459:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594462:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594465:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594468:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764535.594476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067847296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764535.594485:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764535.594488:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764535.594490:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764535.594493:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2003 00000100:00080000:3.0:1789764535.594496:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764535.594499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764535.594502:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764535.594510:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067847424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764535.595236:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067846656:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764535.596218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.596473:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa7b80 x1876702067846656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.597168:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067846656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.597182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.597187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.597191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067846656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764535.597201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764535.598757:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.598781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa6300 x1876702067846784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.598808:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067846784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.598819:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.598831:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.598843:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764535.598851:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764535.598873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.598884:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa7800 x1876702067846912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.598895:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067846912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.598903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.598906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.598909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067846912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764535.598924:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764535.598944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.598954:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa5180 x1876702067847040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.598964:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067847040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.598971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.598974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.598977:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764535.598982:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764535.598997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.599005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa4380 x1876702067847168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.599015:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067847168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.599022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.599025:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.599027:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764535.599033:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067847296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764535.599046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.599054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa4a80 x1876702067847296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.599064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067847296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.599072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.599074:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.599077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067847296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764535.599082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067847424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764535.599096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764535.599103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764535/real 1789764535] req@ffff936010aa5880 x1876702067847424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764535.599112:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067847424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764535.599120:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764535.599122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764535.599125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067847424:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764536.362036:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764537.386021:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764538.410027:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764539.434487:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764540.458601:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764540.715015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764540.715033:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715120:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715134:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715143:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764540.715320:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067847552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764540.715359:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715364:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715369:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715373:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715385:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764540.715400:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067847680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764540.715416:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715420:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715423:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715427:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715432:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764540.715446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067847552:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764540.715513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.715553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef4000 x1876702067847552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.715570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067847552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.715582:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.715586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.715589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067847552:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764540.715597:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764540.715615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.715626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef5c00 x1876702067847680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.715637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067847680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.715644:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.715648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.715651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764540.715710:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067847808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715736:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715740:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764540.715742:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715748:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715753:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715758:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715764:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764540.715780:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067847936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715788:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764540.715793:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715858:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715862:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715866:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715871:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764540.715883:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067848064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715893:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715896:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764540.715899:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715902:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715905:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715909:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715913:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764540.715923:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067848192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764540.715931:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764540.715934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764540.715935:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764540.715939:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2008 00000100:00080000:1.0:1789764540.715942:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764540.715946:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764540.715950:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764540.715962:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067848320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764540.717690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764540.717933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.717955:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef5880 x1876702067847808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.717970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067847808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.717982:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.717987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.717991:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764540.718000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764540.718021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.718033:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef4700 x1876702067847936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.718156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067847936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.718169:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.718173:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.718176:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067847936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764540.718183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764540.718206:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.718216:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef5180 x1876702067848064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.718227:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067848064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.718234:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.718237:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.718239:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764540.718245:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067848192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764540.718334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.718348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef4a80 x1876702067848192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.718359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067848192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.718366:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.718516:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.718520:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067848192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764540.718528:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067848320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764540.718551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764540.718564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764540/real 1789764540] req@ffff936010ef5c00 x1876702067848320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764540.718580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067848320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764540.718588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764540.718591:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764540.718594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067848320:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764541.482473:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764542.506176:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764543.530045:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764544.557363:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764545.583334:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764545.834986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.834999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764545.835003:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835013:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835024:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835033:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764545.835070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067848448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764545.835106:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835110:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835114:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835118:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835128:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764545.835139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067848576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764545.835155:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835159:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835162:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835166:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835170:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764545.835180:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067848704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835192:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764545.835194:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835197:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835200:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835204:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764545.835218:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067848832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835227:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764545.835231:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835235:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835238:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835242:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835246:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764545.835255:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067848960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835267:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764545.835268:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835272:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835275:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835278:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835283:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764545.835293:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067849088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764545.835302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764545.835306:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764545.835307:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764545.835311:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2013 00000100:00080000:3.0:1789764545.835314:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764545.835318:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764545.835322:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764545.835331:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067849216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764545.835424:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067848448:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764545.835481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835519:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa5c00 x1876702067848448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067848448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835545:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835548:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835552:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067848448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764545.835559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764545.835579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835590:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa4000 x1876702067848576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835601:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067848576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764545.835628:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764545.835646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa7100 x1876702067848704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835666:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067848704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835674:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835678:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835681:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764545.835686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764545.835703:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa4700 x1876702067848832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835725:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067848832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835735:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764545.835756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764545.835783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa6a00 x1876702067848960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835830:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067848960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835838:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835841:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835844:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067848960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764545.835850:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764545.835869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835881:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa5500 x1876702067849088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835892:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067849088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764545.835910:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764545.835929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764545.835940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764545/real 1789764545] req@ffff936010aa7480 x1876702067849216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764545.835951:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067849216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764545.835958:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764545.835961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764545.835964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849216:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764546.604929:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764547.626292:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764548.650696:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764549.674216:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764550.699981:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764550.960651:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.960662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764550.960666:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.960676:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.960685:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.960691:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764550.960721:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067849344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764550.960819:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067849344:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764550.960885:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764550.961416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.961420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764550.961423:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.961427:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.961431:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.961443:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.961452:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764550.961466:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067849472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764550.962621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020dac700 x1876702067849344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.962645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067849344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.962655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.962659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.962663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067849344:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764550.962685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764550.962734:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764550.962747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020daf100 x1876702067849472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.962758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067849472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.962765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.962768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.962771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849472:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764550.965735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.965745:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764550.965748:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.965757:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.965768:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.965776:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.965786:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764550.966412:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067849600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764550.975402:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764550.975486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764550.975687:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.975692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764550.975695:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.975700:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.975707:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.975713:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.975732:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764550.975758:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067849728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764550.977678:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020dac380 x1876702067849600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.977704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067849600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.977716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.977721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.985597:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764550.985637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764550.985706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764550.985747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020dad180 x1876702067849728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.985765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067849728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.985780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.985784:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.985787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849728:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764550.986420:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.986426:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764550.986429:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.986434:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.987348:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.987357:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.987436:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764550.987474:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067849856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764550.987490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.987494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764550.987496:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.987500:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.987504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.987508:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.987513:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764550.987523:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067849984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764550.987537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764550.987540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764550.987542:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764550.987545:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2018 00000100:00080000:0.0:1789764550.987549:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764550.987552:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764550.987556:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764550.987566:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067850112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764550.987681:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764550.987734:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764550.987752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020dae300 x1876702067849856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.987767:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067849856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.987776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.987780:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.987783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067849856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764550.987789:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764550.987838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764550.987849:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020daf800 x1876702067849984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.987860:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067849984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.987867:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.987870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.987874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067849984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764550.987879:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067850112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764550.987896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764550.987905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764550/real 1789764550] req@ffff936020dadc00 x1876702067850112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764550.987916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067850112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764550.987923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764550.988245:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764550.988251:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067850112:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764551.722298:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764552.752498:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764553.772599:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764554.797452:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764555.818867:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764556.074035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764556.074050:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074112:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074123:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074131:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764556.074165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067850240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074189:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764556.074194:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074203:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764556.074211:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764556.074222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067850368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074231:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074234:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764556.074236:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074245:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764556.074249:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764556.074258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067850496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764556.074273:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074277:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074280:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074283:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764556.074287:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764556.074295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067850624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:2.0:1789764556.074309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067850240:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764556.074312:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074320:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074323:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074326:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764556.074330:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764556.074340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067850752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074354:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074357:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764556.074359:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074363:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074367:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074370:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764556.074373:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764556.074378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764556.074386:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067850880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764556.074396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764556.074400:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764556.074401:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764556.074405:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2023 00000100:00080000:0.0:1789764556.074408:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764556.074411:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764556.074420:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020dace00 x1876702067850240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.074452:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067850240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.074463:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.074466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.074478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067850240:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764556.074486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764556.074965:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.074983:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020dadf80 x1876702067850368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.074997:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067850368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.075006:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.075009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.075012:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764556.075019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850496:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789764556.075040:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764556.075043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.075059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020daea00 x1876702067850496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.075244:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067850496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.075254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.075257:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.075261:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764556.075298:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764556.075324:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.075337:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020dadc00 x1876702067850624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.075350:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067850624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.075358:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789764556.075363:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067851008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764556.075363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.075369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764556.075380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764556.075404:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.075415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020daf800 x1876702067850752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.075428:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067850752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.075643:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.075647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.075650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067850752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764556.075657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067850880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764556.075682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.076619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020dae300 x1876702067850880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.076638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067850880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.076646:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.076650:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.076654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067850880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764556.076677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764556.078068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764556.078984:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764556/real 1789764556] req@ffff936020dad180 x1876702067851008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764556.079002:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067851008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764556.079012:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764556.079017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764556.079020:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764556.842226:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764557.867135:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764558.890042:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764559.914027:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764560.938274:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764561.193970:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.193982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764561.193986:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764561.193997:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194008:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194015:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764561.194053:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067851136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764561.194084:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.194088:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764561.194090:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764561.194095:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194098:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194102:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194111:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764561.194124:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067851264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764561.194135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.194139:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764561.194141:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789764561.194146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067851136:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764561.194149:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194156:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194161:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194166:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764561.194179:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067851392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764561.194189:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.194193:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764561.194194:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764561.194198:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194202:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194206:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764561.194219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764561.194224:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067851520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764561.194235:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.194238:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764561.194240:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764561.194244:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194248:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194251:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194257:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764561.194264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020dac380 x1876702067851136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764561.194270:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067851648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764561.194289:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067851136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764561.194308:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764561.194311:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764561.194313:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764561.194317:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764561.194318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067851136:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764561.194330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764561.194352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764561.194364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020daf100 x1876702067851264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764561.194376:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067851264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194383:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764561.194394:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00100000:2.0:1789764561.194396:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851264:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764561.194401:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194409:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194415:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764561.194426:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764561.194431:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067851776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764561.194443:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764561.194446:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764561.194448:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789764561.194453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764561.194454:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2029 00000100:00080000:0.0:1789764561.194460:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764561.194464:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764561.194468:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764561.194473:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020dac700 x1876702067851392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764561.194480:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067851904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764561.194498:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067851392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194507:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194511:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764561.194534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764561.194556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764561.194568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020dafb80 x1876702067851520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764561.194579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067851520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194591:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764561.194598:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764561.194618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764561.194628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020dac000 x1876702067851648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764561.194638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067851648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067851648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764561.194674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764561.194694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764561.194705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020dae680 x1876702067851776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764561.194715:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067851776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194723:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194726:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764561.194735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764561.194753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764561.194762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764561/real 1789764561] req@ffff936020daed80 x1876702067851904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764561.194773:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067851904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764561.194781:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764561.194784:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764561.194787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067851904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764561.962299:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764562.987325:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764564.010603:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764565.033998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764566.059855:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764566.314052:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314063:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764566.314067:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314078:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314088:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314095:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764566.314130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067852032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314169:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764566.314175:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314179:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314182:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314185:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314196:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764566.314207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067852160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314216:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764566.314221:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314224:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314227:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314231:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314235:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764566.314245:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067852288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764566.314258:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314261:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314264:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314267:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314271:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764566.314282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067852416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764566.314293:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314296:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314298:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314302:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764566.314314:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067852544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764566.314326:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314329:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314332:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314336:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314339:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764566.314347:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067852672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764566.314360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764566.314362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764566.314364:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764566.314367:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2034 00000100:00080000:1.0:1789764566.314371:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764566.314374:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764566.314378:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764566.314386:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067852800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764566.314720:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067852032:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764566.314794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.314865:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef4000 x1876702067852032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.314882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067852032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.314892:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.314896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.314900:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067852032:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764566.314912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764566.314933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.314946:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef4380 x1876702067852160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.314969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067852160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.314977:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.314980:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.314983:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764566.314990:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764566.315019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.315030:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef7100 x1876702067852288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.315043:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067852288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.315051:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.315054:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.315057:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764566.315062:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764566.315081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.315090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef4e00 x1876702067852416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.315101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067852416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.315108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.315112:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.315116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764566.315121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764566.315138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.315147:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef6d80 x1876702067852544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.315157:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067852544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.315164:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.315167:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.315170:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067852544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764566.315174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067852672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764566.315191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.315200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef6300 x1876702067852672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.315216:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067852672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.315223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.315226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.315229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067852672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764566.315234:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067852800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764566.315253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764566.315264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764566/real 1789764566] req@ffff936010ef7b80 x1876702067852800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764566.315275:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067852800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764591 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764566.315283:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764566.315286:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764566.315290:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067852800:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764567.082213:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764568.106008:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764569.130226:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764570.154429:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764571.178208:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764571.434496:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434506:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764571.434510:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434520:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434529:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434536:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764571.434583:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067852928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434610:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434614:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764571.434616:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434619:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434622:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434626:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434634:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764571.434644:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067853056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434654:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434656:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764571.434658:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434661:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434664:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434667:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434670:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764571.434677:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067853184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434685:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434688:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764571.434689:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434692:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434695:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434698:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434702:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764571.434709:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067853312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434717:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434720:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764571.434722:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434724:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434726:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434729:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434732:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764571.434739:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067853440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434746:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434749:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764571.434750:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434753:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434755:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434758:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434761:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764571.434768:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067853568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764571.434776:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764571.434778:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764571.434780:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764571.434783:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2039 00000100:00080000:0.0:1789764571.434786:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764571.434789:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764571.434792:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764571.434838:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067853696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764571.434990:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067852928:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764571.435056:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020dad880 x1876702067852928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067852928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435117:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.435121:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067852928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764571.435128:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764571.435145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435154:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020dad500 x1876702067853056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067853056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435284:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.435338:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764571.435344:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764571.435365:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435376:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020daf480 x1876702067853184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067853184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.435400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764571.435406:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764571.435425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435437:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020daed80 x1876702067853312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435447:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067853312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435459:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435462:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.435465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764571.435470:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764571.435486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020dae680 x1876702067853440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067853440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.435517:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764571.435522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067853568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764571.435538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.435548:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020dac000 x1876702067853568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.435558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067853568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.435564:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.435567:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.436102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067853568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764571.436114:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067853696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764571.436139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764571.436152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764571/real 1789764571] req@ffff936020dafb80 x1876702067853696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764571.436164:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067853696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764571.436237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764571.436240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764571.436244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067853696:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764572.202619:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764573.227504:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764574.250603:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764575.274017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764576.298055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764576.554006:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554019:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764576.554022:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554034:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554043:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554050:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764576.554087:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067853824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554120:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554123:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764576.554125:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554129:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554132:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554136:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554144:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764576.554156:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067853952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554165:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554180:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764576.554181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554185:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554188:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554191:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764576.554206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067854080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554221:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764576.554223:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554229:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554232:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554236:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554240:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764576.554248:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067854208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554257:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554260:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764576.554261:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554265:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554268:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554271:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554275:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764576.554282:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067854336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554290:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554293:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764576.554295:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554298:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554300:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554303:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554307:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764576.554316:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067854464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764576.554324:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764576.554327:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764576.554328:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764576.554331:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2044 00000100:00080000:0.0:1789764576.554334:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764576.554337:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764576.554341:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764576.554349:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067854592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764576.554377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067853824:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764576.554441:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.554482:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020dac700 x1876702067853824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.554499:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067853824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.554510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.554514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.554518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067853824:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764576.554526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764576.554565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.554578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020daf100 x1876702067853952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.554590:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067853952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.554598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.554601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.554604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067853952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764576.554610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764576.554630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.554642:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020dac380 x1876702067854080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.554653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067854080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.554661:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.554664:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.554667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764576.554673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764576.554693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.554705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020dad180 x1876702067854208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.554717:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067854208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.554725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.554728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.554731:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764576.554737:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764576.554756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.554768:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020dae300 x1876702067854336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.554778:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067854336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.555333:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.555339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.555343:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764576.555350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067854464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764576.555378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.555392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020daf800 x1876702067854464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.555404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067854464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.555418:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.555421:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.555424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067854464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764576.555430:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067854592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764576.555452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764576.555463:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764576/real 1789764576] req@ffff936020dadc00 x1876702067854592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764576.555474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067854592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764576.555481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764576.555484:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764576.555487:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067854592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764577.323281:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764578.346025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764579.370021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764580.394044:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764581.418162:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764581.679070:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679083:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764581.679087:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679099:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679109:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679116:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764581.679154:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067854720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764581.679196:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679200:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679203:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679207:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679215:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764581.679226:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067854848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764581.679238:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679249:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764581.679256:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067854976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764581.679266:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679269:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679271:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679273:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764581.679282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067855104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764581.679291:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679294:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679296:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679298:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679301:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764581.679307:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067855232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764581.679316:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679319:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679320:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679323:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679326:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764581.679333:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067855360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764581.679339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764581.679342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764581.679343:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764581.679346:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2049 00000100:00080000:2.0:1789764581.679348:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764581.679350:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764581.679354:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764581.679359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067855488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764581.679447:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067854720:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764581.679521:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061dd500 x1876702067854720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679588:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067854720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679603:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679610:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067854720:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764581.679616:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764581.679637:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679648:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061df480 x1876702067854848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067854848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679668:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679671:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764581.679676:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764581.679694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679704:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061df100 x1876702067854976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679714:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067854976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679725:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679727:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067854976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764581.679733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764581.679753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061dd880 x1876702067855104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067855104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679781:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679787:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764581.679792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764581.679845:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679856:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061ddc00 x1876702067855232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067855232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679873:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679876:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764581.679885:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067855360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764581.679903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061de300 x1876702067855360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067855360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679936:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679939:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067855360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764581.679944:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067855488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764581.679962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764581.679972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764581/real 1789764581] req@ffff9360061ddf80 x1876702067855488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764581.679982:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067855488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764581.679988:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764581.679990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764581.679993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067855488:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764582.442065:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764583.466145:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764584.490081:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764585.514021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764586.538023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764586.793981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.793990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764586.793994:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794004:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794014:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794022:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764586.794054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067855616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764586.794086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794089:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764586.794091:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794094:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794097:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794100:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794108:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764586.794118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067855744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764586.794127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764586.794130:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794136:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794139:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794143:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764586.794152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067855872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764586.794160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794164:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764586.794165:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794168:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794172:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764586.794177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067855616:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764586.794178:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794184:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764586.794197:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067856000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764586.794207:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794210:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764586.794212:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794215:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794218:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794222:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794225:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764586.794236:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067856128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764586.794246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.794327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa4e00 x1876702067855616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.794347:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067855616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.794358:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.794362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.794366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067855616:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764586.794373:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764586.794395:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.794409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa6680 x1876702067855744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.794421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067855744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.794430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.794433:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.794437:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764586.794444:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764586.794478:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.794490:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa5f80 x1876702067855872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.794501:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067855872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.794509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.794513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.794517:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067855872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764586.794546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856000:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764586.794716:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794721:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764586.794723:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794729:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2054 00000800:00000100:0.0:1789764586.794732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764586.794735:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794741:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794747:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764586.794760:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa7480 x1876702067856000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.794774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067856000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.794782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.794785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.794788:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764586.794794:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764586.794846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.794857:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa5500 x1876702067856128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.794868:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067856128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.794876:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.794880:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.794883:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764586.794919:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067856256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764586.794937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764586.794940:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764586.794942:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764586.794946:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2054 00000100:00080000:3.0:1789764586.794950:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764586.794954:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764586.794959:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764586.794970:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067856384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764586.795056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067856256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764586.795082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.795094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa6a00 x1876702067856256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.795106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067856256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.795113:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.795116:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.795119:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067856256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764586.795125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067856384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764586.795144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764586.795155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764586/real 1789764586] req@ffff936010aa4700 x1876702067856384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764586.795165:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067856384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764586.795179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764586.795182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764586.795185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067856384:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764587.562078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764588.585931:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764589.610031:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764590.634586:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764591.658043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764591.914628:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914652:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764591.914656:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914674:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914685:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914692:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764591.914731:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067856512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914761:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914765:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764591.914766:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914770:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914774:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914778:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.914787:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764591.914825:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067856640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914836:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914840:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764591.914842:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914846:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914849:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914853:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.914857:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764591.914877:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067856768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914885:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914888:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764591.914890:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914893:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914896:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914899:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.914903:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764591.914914:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067856896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914922:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914925:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764591.914926:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914929:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914932:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914935:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.914940:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764591.914948:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067857024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914961:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914964:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764591.914965:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.914969:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.914972:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.914975:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.914979:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764591.914987:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067857152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764591.914994:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764591.914998:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764591.914999:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764591.915002:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2059 00000100:00080000:0.0:1789764591.915005:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764591.915008:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764591.915012:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764591.915019:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067857280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764591.915086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067856512:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764591.915144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020daea00 x1876702067856512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915202:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067856512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915212:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067856512:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764591.915227:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764591.915249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020dadf80 x1876702067856640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067856640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915284:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915288:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764591.915293:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764591.915313:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020dace00 x1876702067856768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067856768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915341:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915344:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915348:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764591.915353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764591.915373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915384:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020dadc00 x1876702067856896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915396:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067856896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915403:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915406:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915409:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067856896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764591.915415:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764591.915447:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915459:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020daf800 x1876702067857024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067857024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915477:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915479:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915482:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764591.915488:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067857152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764591.915508:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915519:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020dae300 x1876702067857152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067857152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915545:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067857152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764591.915554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067857280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764591.915573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764591.915584:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764591/real 1789764591] req@ffff936020dad180 x1876702067857280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764591.915596:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067857280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764591.915603:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764591.915606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764591.915609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067857280:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764592.687145:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764593.706622:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764594.730678:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764595.754609:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764596.778361:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764597.034102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764597.034119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034129:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034140:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034146:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764597.034180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067857408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764597.034216:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034220:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034223:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034227:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034236:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764597.034247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067857536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764597.034261:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034265:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034269:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034272:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764597.034286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067857664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764597.034298:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034301:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034305:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034308:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034312:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764597.034320:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067857792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764597.034332:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034335:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034337:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034340:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034344:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764597.034351:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067857920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034361:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764597.034363:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034365:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034368:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034371:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034374:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764597.034381:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067858048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764597.034405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764597.034408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764597.034409:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764597.034412:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2064 00000100:00080000:3.0:1789764597.034415:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764597.034418:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764597.034423:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764597.034444:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067858176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764597.034684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067857408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764597.034781:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764597.034828:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764597.034849:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764597.034869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764597.034889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764597.034911:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764597.802095:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764598.833750:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764599.851106:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764600.107146:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764600.107203:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764600.107220:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764600.128821:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa7100 x1876702067857408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764622 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764600.128856:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067857408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764622 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764600.128868:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764600.128873:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764600.128876:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067857408:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764600.132370:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764600.132410:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764600.132419:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764600.132426:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764600.132433:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764600.132447:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764600.132512:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa4000 x1876702067857536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067857536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132548:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764600.132564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa5c00 x1876702067857664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067857664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132595:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764600.132601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa5880 x1876702067857792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132612:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067857792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132619:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764600.132630:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa4a80 x1876702067857920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067857920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067857920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764600.132660:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa4380 x1876702067858048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067858048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764600.132701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764597/real 1789764600] req@ffff936010aa5180 x1876702067858176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764600.132712:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067858176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764600.132718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764600.132721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764600.132724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764600.881924:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764601.898337:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764602.155942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.155954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764602.155959:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.155971:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.155981:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.155988:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764602.156037:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067858304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156075:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764602.156077:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156089:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156106:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764602.156118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067858432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764602.156132:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156139:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156143:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156147:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764602.156158:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067858560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156166:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156169:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764602.156170:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156173:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156176:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156179:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156183:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764602.156191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067858688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156202:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764602.156204:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156207:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156209:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156212:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156216:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764602.156224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067858816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156232:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156235:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764602.156236:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156240:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156243:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156250:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764602.156258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067858944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764602.156267:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764602.156269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764602.156271:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764602.156275:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2069 00000100:00080000:3.0:1789764602.156279:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764602.156282:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764602.156286:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764602.156295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067859072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764602.157477:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067858304:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764602.157545:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157598:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa5180 x1876702067858304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067858304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157628:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067858304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764602.157640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764602.157664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157678:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa4380 x1876702067858432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157688:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067858432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157696:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764602.157708:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764602.157727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157739:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa4a80 x1876702067858560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067858560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764602.157771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764602.157791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157832:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa5880 x1876702067858688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157844:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067858688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157864:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764602.157869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764602.157889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa5c00 x1876702067858816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157910:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067858816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157917:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157921:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157924:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067858816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764602.157930:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764602.157948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.157959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa4000 x1876702067858944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.157969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067858944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.157981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.157985:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.157988:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067858944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764602.157994:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764602.158012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764602.158023:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764602/real 1789764602] req@ffff936010aa7800 x1876702067859072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764602.158033:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067859072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764602.158040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764602.158042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764602.158045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859072:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764602.923322:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764603.947439:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764604.971038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764605.997312:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764607.018950:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764607.274411:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.274425:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764607.274429:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.274440:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.274451:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.274458:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764607.274495:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067859200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764607.274558:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067859200:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764607.274632:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764607.274730:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.274735:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764607.274736:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.274741:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.274746:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.274751:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.274760:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764607.274774:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067859328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764607.275185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef7480 x1876702067859200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.275204:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067859200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.275214:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.275218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.275222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067859200:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764607.275246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764607.275271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764607.275283:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef7800 x1876702067859328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.275294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067859328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.275359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.275364:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.275367:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859328:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764607.276249:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.276256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764607.276259:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.276265:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.276271:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.276277:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.276286:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764607.276307:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067859456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764607.276358:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764607.276394:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764607.276454:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.276459:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764607.276460:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.276464:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.276468:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.276472:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.276477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764607.276499:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067859584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764607.276694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef5500 x1876702067859456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.276710:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067859456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.276720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.276723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.276727:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764607.276745:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764607.276772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764607.276783:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef6a00 x1876702067859584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.276794:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067859584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.276821:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.276824:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.276828:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859584:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764607.276887:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.276892:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764607.276894:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.276898:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.276902:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.276906:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.276911:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764607.276924:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067859712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764607.276934:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.276936:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764607.276938:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.276940:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.276943:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.276947:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.276950:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764607.276959:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067859840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764607.276967:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764607.276970:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764607.276971:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764607.276975:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2075 00000100:00080000:1.0:1789764607.276977:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764607.276981:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764607.276986:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764607.276995:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067859968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764607.277057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764607.277086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764607.277098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef6680 x1876702067859712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.277110:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067859712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.277117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.277120:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.277123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067859712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764607.277128:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764607.277148:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764607.277159:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef5f80 x1876702067859840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.277171:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067859840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.277223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.277226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.277229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764607.277235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764607.277254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764607.277264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764607/real 1789764607] req@ffff936010ef6a00 x1876702067859968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764607.277275:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067859968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764632 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764607.277281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764607.277283:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764607.277286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067859968:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764608.042077:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764609.066003:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764610.090317:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764611.115584:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764612.138063:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764612.394022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394033:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764612.394037:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394047:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394057:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394064:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764612.394101:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067860096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394135:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394139:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764612.394141:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394145:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394148:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394152:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394161:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764612.394172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067860224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764612.394186:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394190:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394193:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394196:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394200:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764612.394208:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067860352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394216:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764612.394221:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394224:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394227:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394230:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394234:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764612.394243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067860480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764612.394255:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394258:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394261:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394265:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394269:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764612.394291:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067860608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764612.394307:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394310:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394313:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394316:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394319:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764612.394328:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067860736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764612.394336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764612.394339:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764612.394341:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764612.394344:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2080 00000100:00080000:1.0:1789764612.394347:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764612.394350:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764612.394354:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764612.394363:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067860864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764612.394833:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067860096:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764612.394891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.394944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef5500 x1876702067860096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.394960:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067860096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.394970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.394973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.394977:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067860096:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764612.394984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764612.395003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef7800 x1876702067860224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395031:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067860224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395039:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764612.395051:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764612.395067:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395077:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef7480 x1876702067860352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067860352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395095:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395098:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395101:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764612.395107:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764612.395124:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395133:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef7b80 x1876702067860480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395143:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067860480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395159:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764612.395168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764612.395186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395195:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef6300 x1876702067860608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067860608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395213:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395220:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067860608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764612.395228:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067860736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764612.395246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395256:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef6d80 x1876702067860736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067860736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395284:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067860736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764612.395298:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067860864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764612.395319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764612.395329:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764612/real 1789764612] req@ffff936010ef4e00 x1876702067860864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764612.395340:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067860864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764637 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764612.395347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764612.395350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764612.395353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067860864:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764613.163169:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764614.186627:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764615.210089:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764616.234056:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764617.258253:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764617.514765:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.514779:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764617.514783:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.514793:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.514926:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.514934:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764617.515248:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067860992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764617.515314:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067860992:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764617.515370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764617.515399:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.515404:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764617.515406:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.515410:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.515414:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.515417:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764617.515427:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764617.515442:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067861120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764617.522214:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020dac380 x1876702067860992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764617.522262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067860992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764617.522274:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764617.522279:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764617.522283:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067860992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764617.522316:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764617.522371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764617.522388:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020daf100 x1876702067861120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764617.522400:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067861120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764617.522408:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764617.522411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764617.522414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861120:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764617.522581:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.522586:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764617.522588:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.522593:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.522599:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.522606:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764617.522616:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764617.522640:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067861248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764617.522787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764617.522825:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764617.522926:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.522930:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764617.522932:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.522937:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.522941:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.522944:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764617.523864:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764617.523885:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067861376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764617.528393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020dac700 x1876702067861248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764617.528430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067861248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764617.528449:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764617.528454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764617.528459:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764617.528490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764617.528551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764617.528568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020dafb80 x1876702067861376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764617.528595:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067861376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764617.528604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764617.528608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764617.528612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861376:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764617.528787:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.528793:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764617.528795:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.528811:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.528816:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.528823:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764617.528832:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764617.528856:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067861504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764617.528869:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.528872:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764617.528874:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.528878:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.528882:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.528885:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764617.528985:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764617.529009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764617.529022:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020dac000 x1876702067861504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764617.529034:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067861504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764617.529042:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764617.529044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764617.529048:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067861504:192.168.203.137@tcp:8:kworker.0 00000008:00100000:0.0:1789764617.529092:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764617.529104:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067861632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764617.529141:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067861632:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764617.529220:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764617.529224:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764617.529226:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764617.529230:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2085 00000100:00080000:0.0:1789764617.529234:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764617.529250:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764617.529255:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764617.529265:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067861760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764617.529364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764617.529569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020dae680 x1876702067861632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764617.529639:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067861632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764617.529648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764617.529651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764617.529655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067861632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764617.529669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067861760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764617.529693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764617.529705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764617/real 1789764617] req@ffff936020daed80 x1876702067861760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764617.529717:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067861760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764617.529724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764617.529727:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764617.529730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067861760:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764618.281985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764619.306352:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764620.334625:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764621.354898:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764622.378061:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764622.634031:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634044:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764622.634049:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634062:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634077:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634085:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764622.634137:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067861888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634174:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764622.634181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634186:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634190:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634193:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764622.634229:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067862016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634239:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634243:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764622.634245:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634249:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634252:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634256:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634261:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764622.634272:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067862144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634282:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634285:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764622.634287:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634292:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634295:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634299:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634304:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764622.634314:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067862272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634323:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634326:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764622.634328:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634331:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634334:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634338:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634343:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764622.634353:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067862400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634374:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634377:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764622.634379:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634383:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634386:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634390:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634395:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764622.634407:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067862528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764622.634416:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764622.634419:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764622.634420:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764622.634423:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2090 00000100:00080000:0.0:1789764622.634427:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764622.634431:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764622.634435:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764622.634445:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067862656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764622.634563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067861888:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764622.634655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.634719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020daed80 x1876702067861888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.634739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067861888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.634750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.634754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.634758:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067861888:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764622.634765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764622.634791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.634835:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020dae680 x1876702067862016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.634848:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067862016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.634857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.634860:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.634863:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764622.634869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764622.634890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.634901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020daf480 x1876702067862144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.634913:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067862144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.634920:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.634924:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.634927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764622.634933:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764622.634953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.634963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020dad500 x1876702067862272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.634975:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067862272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.634982:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.634985:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.634988:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764622.634994:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764622.635015:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.635026:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020dad880 x1876702067862400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.635037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067862400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.635044:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.635047:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.635051:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764622.635056:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067862528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764622.635076:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.635087:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020dac000 x1876702067862528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.635098:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067862528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.635105:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.635108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.635112:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067862528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764622.635117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067862656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764622.635137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764622.635147:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764622/real 1789764622] req@ffff936020dafb80 x1876702067862656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764622.635158:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067862656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764622.635166:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764622.635169:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764622.635172:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067862656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764623.402647:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764624.428605:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764625.452822:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764626.474437:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764627.498981:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764627.753990:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754002:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764627.754006:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754016:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754026:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754034:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764627.754071:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067862784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754109:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754113:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764627.754114:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754119:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754122:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754127:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754135:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764627.754148:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067862912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754157:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754160:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764627.754171:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754175:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754178:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754182:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754186:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764627.754198:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067863040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754209:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764627.754211:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754214:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754217:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754221:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754225:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764627.754235:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067863168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764627.754257:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754260:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754263:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754266:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754276:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764627.754286:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067863296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754294:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754298:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764627.754299:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754303:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754306:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754308:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754312:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764627.754321:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067863424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764627.754329:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764627.754332:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764627.754334:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764627.754337:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2095 00000100:00080000:2.0:1789764627.754340:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764627.754344:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764627.754349:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764627.754357:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067863552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764627.754431:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067862784:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764627.754501:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754540:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061dc380 x1876702067862784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067862784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754567:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754571:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067862784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764627.754581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764627.754603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061dc700 x1876702067862912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754627:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067862912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067862912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764627.754652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764627.754670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061ddf80 x1876702067863040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067863040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754712:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754718:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764627.754723:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764627.754742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061de300 x1876702067863168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067863168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754773:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764627.754782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764627.754820:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754832:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061ddc00 x1876702067863296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754843:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067863296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754851:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754857:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764627.754862:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067863424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764627.754880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754891:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061dd880 x1876702067863424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754900:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067863424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754907:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754910:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067863424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764627.754918:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067863552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764627.754937:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764627.754948:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764627/real 1789764627] req@ffff9360061df100 x1876702067863552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764627.754958:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067863552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764627.754966:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764627.754969:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764627.754972:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067863552:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764628.522145:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764629.546864:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764630.569985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764631.593990:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764632.619145:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764632.875057:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875069:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764632.875073:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875083:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875093:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875101:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764632.875150:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067863680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875183:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764632.875188:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875192:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875196:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875200:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764632.875221:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067863808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875234:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764632.875235:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875240:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875243:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875246:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875250:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764632.875260:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067863936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875269:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875272:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764632.875274:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875277:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875280:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875283:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875287:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764632.875297:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067864064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875305:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875308:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764632.875309:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875312:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875314:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875318:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875322:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764632.875330:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067864192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875339:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875342:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764632.875344:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875347:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875349:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875353:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875356:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764632.875365:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067864320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764632.875372:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764632.875375:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764632.875377:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764632.875380:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2100 00000100:00080000:1.0:1789764632.875383:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764632.875386:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764632.875390:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764632.875398:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067864448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764632.877743:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067863680:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764632.877862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878085:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef7100 x1876702067863680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878104:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067863680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878115:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067863680:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764632.878145:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764632.878175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878191:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef4380 x1876702067863808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067863808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878214:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764632.878229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764632.878253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef4000 x1876702067863936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067863936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067863936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764632.878297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764632.878315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef5c00 x1876702067864064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878333:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067864064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878346:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878349:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764632.878358:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764632.878378:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef4a80 x1876702067864192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067864192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878410:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878412:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764632.878418:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067864320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764632.878511:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef5180 x1876702067864320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878613:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067864320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878621:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878625:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878630:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067864320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764632.878637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067864448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764632.878659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764632.878670:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764632/real 1789764632] req@ffff936010ef4700 x1876702067864448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764632.878683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067864448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764632.878689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764632.878692:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764632.878695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067864448:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764633.642907:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764634.666010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764635.690576:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764636.716439:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764637.738147:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764637.994044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994055:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764637.994059:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994070:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994079:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994095:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764637.994130:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067864576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994161:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764637.994173:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994177:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994193:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764637.994205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067864704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764637.994219:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994223:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994227:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994231:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994235:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764637.994247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067864832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764637.994261:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994264:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994268:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994272:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764637.994286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067864960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764637.994298:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994301:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994304:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994308:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994312:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764637.994326:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067865088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994334:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994337:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764637.994338:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994342:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994345:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994348:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994353:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764637.994361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067865216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764637.994375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764637.994378:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764637.994379:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764637.994383:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2105 00000100:00080000:3.0:1789764637.994387:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764637.994390:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764637.994395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764637.994404:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067865344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764637.994889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067864576:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764637.995001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995134:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa6300 x1876702067864576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067864576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995221:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067864576:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764637.995230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764637.995254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa7b80 x1876702067864704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995278:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067864704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995285:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764637.995297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764637.995315:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa7800 x1876702067864832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067864832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995342:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995345:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995348:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764637.995353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764637.995369:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa4000 x1876702067864960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995388:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067864960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995395:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067864960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764637.995405:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764637.995421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995431:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa5c00 x1876702067865088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995440:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067865088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995447:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995450:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995453:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764637.995458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067865216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764637.995476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa5880 x1876702067865216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067865216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067865216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764637.995521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067865344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764637.995538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764637.995548:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764637/real 1789764637] req@ffff936010aa4a80 x1876702067865344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764637.995726:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067865344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764637.995734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764637.995737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764637.995739:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067865344:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764638.762059:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764639.786286:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764640.820312:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764641.834018:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764642.860674:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764643.115259:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115271:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764643.115275:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115286:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115296:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115303:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764643.115340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067865472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764643.115375:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115378:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115382:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115386:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764643.115407:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067865600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115416:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115419:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764643.115420:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115424:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115427:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115430:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115435:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764643.115445:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067865728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764643.115457:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115460:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115463:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115466:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115470:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764643.115479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067865856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764643.115496:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115500:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115503:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115506:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115510:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764643.115531:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067865984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115539:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115543:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764643.115544:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115547:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115554:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115558:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764643.115569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067866112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764643.115578:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764643.115581:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764643.115582:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764643.115586:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2110 00000100:00080000:3.0:1789764643.115589:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764643.115593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764643.115597:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764643.115606:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067866240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764643.117333:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067865472:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764643.117404:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa4380 x1876702067865472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117480:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067865472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117497:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067865472:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764643.117513:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764643.117537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117549:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa5180 x1876702067865600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067865600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117568:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117572:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764643.117582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764643.117601:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa7100 x1876702067865728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067865728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117630:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117633:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117636:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764643.117643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764643.117663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa4700 x1876702067865856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067865856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117691:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764643.117703:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764643.117721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa6a00 x1876702067865984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067865984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067865984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764643.117761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067866112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764643.117779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa5500 x1876702067866112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117852:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067866112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117860:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067866112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764643.117902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067866240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764643.117922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764643.117932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764643/real 1789764643] req@ffff936010aa7480 x1876702067866240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764643.117942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067866240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764643.117951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764643.117954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764643.117957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067866240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764643.882555:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764644.906252:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764645.930294:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764646.955475:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764647.980130:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764648.234410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764648.234424:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234433:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234444:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234452:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764648.234727:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067866368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234764:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764648.234766:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234770:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234774:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234779:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.234787:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764648.234845:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067866496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234859:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764648.234861:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234865:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234868:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234872:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.234876:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764648.234886:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067866624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234897:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764648.234898:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234901:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234904:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234907:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.234911:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764648.234919:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067866752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764648.234932:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234935:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234938:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234941:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.234944:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764648.234953:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067866880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.234964:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764648.234965:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.234968:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.234971:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.234975:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.234980:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764648.234988:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067867008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764648.234997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764648.235000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764648.235001:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764648.235005:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2116 00000100:00080000:0.0:1789764648.235008:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764648.235024:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764648.235029:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764648.235039:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067867136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764648.235453:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067866368:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764648.235569:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020dac700 x1876702067866368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235632:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067866368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235642:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235645:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067866368:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764648.235656:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764648.235675:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235684:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020daf100 x1876702067866496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235695:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067866496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235703:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764648.235714:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764648.235728:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020dac380 x1876702067866624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067866624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235756:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235759:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764648.235764:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764648.235778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235786:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020dad180 x1876702067866752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067866752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235823:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235826:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235829:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764648.235834:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764648.235849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235859:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020dae300 x1876702067866880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067866880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235896:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067866880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764648.235901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764648.235922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.235932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020daf800 x1876702067867008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.235955:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067867008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.235962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.235965:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.235968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764648.235974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764648.235992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764648.236002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764648/real 1789764648] req@ffff936020dadc00 x1876702067867136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764648.236022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067867136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764673 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764648.236034:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764648.236037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764648.236040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764649.010088:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764650.032052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764651.053478:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764652.076557:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764653.098199:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764653.354535:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.354546:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764653.354551:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.354562:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.354573:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.354581:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764653.354619:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067867264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764653.354693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067867264:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764653.354765:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764653.354824:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.354828:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764653.354830:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.354834:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.354839:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.354843:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.354852:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764653.354866:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067867392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764653.359930:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef5880 x1876702067867264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.359959:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067867264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.359970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.359986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.359990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067867264:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764653.360022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764653.360077:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.360095:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef4700 x1876702067867392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.360108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067867392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.360116:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.360120:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.360123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867392:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764653.360326:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.360331:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764653.360334:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.360340:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.360346:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.360352:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.360362:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764653.360386:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067867520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764653.360405:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.360408:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764653.360409:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.360413:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.360417:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.360420:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.360425:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764653.360435:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067867648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764653.360444:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.360447:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764653.360449:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.360452:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.360456:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.360459:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.360463:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764653.360474:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067867776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764653.360482:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.360485:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764653.360486:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.360489:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.360493:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.360497:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.360501:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764653.360582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764653.360611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.360624:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef5180 x1876702067867520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.360637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067867520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.360645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.360648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.360651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764653.360657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764653.360678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.360691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef4a80 x1876702067867648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.360703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067867648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.360710:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.360713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.360716:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764653.360721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764653.360831:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.360842:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef5c00 x1876702067867776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.360853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067867776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.360860:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.360863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.360866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067867776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764653.360930:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067867904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764653.360951:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764653.360955:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764653.360956:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764653.360961:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2121 00000100:00080000:1.0:1789764653.360965:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764653.360968:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764653.360985:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764653.360996:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067868032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764653.361061:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764653.361085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.361098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef4000 x1876702067867904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.361109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067867904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.361117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.361120:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.361123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067867904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764653.361128:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764653.361147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764653.361157:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764653/real 1789764653] req@ffff936010ef4380 x1876702067868032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764653.361169:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067868032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764678 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764653.361181:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764653.361184:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764653.361187:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868032:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764654.123186:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764655.145988:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764656.170008:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764657.197030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764658.219185:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764658.473917:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.473926:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764658.473931:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.473953:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.473963:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.473970:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764658.474003:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067868160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474035:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474039:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764658.474041:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474045:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474048:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474052:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474061:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764658.474072:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067868288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474082:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474085:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764658.474087:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474091:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474094:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474102:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764658.474103:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067868160:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764658.474113:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067868416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474123:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474126:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764658.474128:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474132:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474135:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474138:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474142:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764658.474150:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067868544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474158:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474160:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764658.474162:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474164:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474167:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474170:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474173:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764658.474180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067868672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474188:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474191:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764658.474192:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474195:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474198:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474201:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764658.474226:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067868800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764658.474234:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764658.474236:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764658.474238:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764658.474241:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2126 00000100:00080000:1.0:1789764658.474244:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764658.474248:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764658.474252:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764658.474260:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067868928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764658.474349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.474387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef7100 x1876702067868160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.474407:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067868160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.474418:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.474422:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.474425:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067868160:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764658.474433:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764658.474455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.474901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef4e00 x1876702067868288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.474918:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067868288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.474927:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.474930:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.474935:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764658.474972:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764658.474995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.475388:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef6d80 x1876702067868416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.475401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067868416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.475409:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.475412:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.475416:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764658.475423:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764658.475445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.475457:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef6300 x1876702067868544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.475538:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067868544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.475547:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.475550:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.475553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764658.475559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764658.475579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.475591:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef7b80 x1876702067868672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.475603:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067868672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.475610:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.475613:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.475616:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067868672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764658.475621:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764658.475636:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.475645:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef7480 x1876702067868800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.475655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067868800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.475662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.475665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.475668:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764658.475673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764658.475688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764658.475697:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764658/real 1789764658] req@ffff936010ef7800 x1876702067868928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764658.475705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067868928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764658.475712:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764658.475715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764658.475717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067868928:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764659.243752:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764660.266000:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764661.290565:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764662.314050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764663.338005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764663.594235:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594245:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764663.594249:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594258:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594268:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594275:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764663.594311:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067869056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594341:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594345:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764663.594346:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594350:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594353:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594357:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594364:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764663.594376:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067869184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594385:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594388:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764663.594389:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594393:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594395:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594398:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594402:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764663.594412:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067869312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594420:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594423:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764663.594424:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594428:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594431:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594434:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594438:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764663.594446:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067869440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594454:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594457:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764663.594458:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594461:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594464:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594467:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594471:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764663.594480:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067869568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594488:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594491:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764663.594492:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594495:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594498:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594501:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594505:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764663.594515:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067869696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764663.594523:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764663.594525:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764663.594527:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764663.594530:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2131 00000100:00080000:2.0:1789764663.594533:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764663.594535:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764663.594539:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764663.594548:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067869824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764663.594687:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067869056:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764663.594780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764663.594832:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764663.594854:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764663.594874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764663.594894:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067869696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764663.594926:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067869824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764664.362826:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764665.392251:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764666.410113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789764666.672723:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789764666.672764:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764666.672779:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672848:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672856:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672863:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672869:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672885:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764666.672893:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764666.675852:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061df480 x1876702067869056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.675897:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067869056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.675908:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.675912:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.675916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067869056:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764666.675924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061dd500 x1876702067869184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.675937:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067869184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.675945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.675957:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.675960:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764666.675967:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061dc000 x1876702067869312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.675978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067869312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.675986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.675989:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.675992:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764666.675999:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061dce00 x1876702067869440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.676010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067869440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.676017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.676020:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.676023:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764666.676029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061dd180 x1876702067869568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.676039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067869568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.676046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.676059:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.676063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067869568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764666.676070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061dca80 x1876702067869696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.676081:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067869696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.676088:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.676092:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.676094:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067869696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764666.676100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764663/real 1789764666] req@ffff9360061df800 x1876702067869824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764666.676120:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067869824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764666.676127:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764666.676130:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764666.676134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067869824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764667.434174:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764668.457985:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764668.714783:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.714792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764668.714920:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.714931:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.714941:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.714948:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764668.714983:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067869952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764668.715055:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715059:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715063:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715067:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715076:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764668.715088:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067870080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764668.715102:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715105:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715111:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715115:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764668.715125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067870208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715137:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764668.715138:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715141:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715145:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715149:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715153:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764668.715163:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067870336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715173:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764668.715174:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715177:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715179:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715182:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764668.715196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067870464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715206:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764668.715207:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715210:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715212:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764668.715227:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067870592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764668.715235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764668.715237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764668.715239:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764668.715242:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2136 00000100:00080000:1.0:1789764668.715244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764668.715247:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764668.715251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764668.715258:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067870720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764668.715954:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067869952:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764668.716355:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.716398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef5500 x1876702067869952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.716416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067869952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.716946:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.716951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.716955:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067869952:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764668.716965:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764668.717012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717027:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef6a00 x1876702067870080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717040:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067870080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717065:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764668.717072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764668.717323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717338:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef5f80 x1876702067870208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717356:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067870208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717364:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764668.717376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764668.717398:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef6680 x1876702067870336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717420:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067870336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717427:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764668.717439:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764668.717463:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717472:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef7800 x1876702067870464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067870464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717498:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717501:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764668.717696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067870592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764668.717720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef7480 x1876702067870592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717744:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067870592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717757:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067870592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764668.717776:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067870720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764668.717796:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764668.717837:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764668/real 1789764668] req@ffff936010ef7b80 x1876702067870720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764668.717849:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067870720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764668.717856:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764668.717859:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764668.717874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067870720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764669.482169:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764670.507750:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764671.531120:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764672.554009:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764673.578218:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764673.834886:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.834896:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764673.834900:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.834907:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.834917:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.834923:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764673.834956:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067870848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.834985:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.834988:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764673.834990:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.834993:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.834996:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835000:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835008:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764673.835019:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067870976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.835027:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.835030:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764673.835032:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.835035:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.835038:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835041:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835045:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764673.835054:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067871104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.835063:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.835065:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764673.835067:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.835070:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.835073:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835076:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835080:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764673.835091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067871232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.835098:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.835101:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764673.835102:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.835106:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.835109:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835111:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835116:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764673.835126:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067871360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.835133:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.835136:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764673.835138:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.835141:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.835144:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835147:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835152:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764673.835161:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067871488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764673.835169:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764673.835172:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764673.835174:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764673.835177:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2141 00000100:00080000:0.0:1789764673.835179:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764673.835182:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764673.835186:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764673.835195:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067871616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764673.835407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067870848:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764673.835464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.835500:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020dace00 x1876702067870848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.835517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067870848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.835527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.835531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.835534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067870848:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764673.835541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764673.835618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.835631:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020dadf80 x1876702067870976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.835642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067870976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.835650:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.835653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.835656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067870976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764673.835662:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764673.835682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.835692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020daea00 x1876702067871104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.835702:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067871104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.835709:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.835712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.835715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764673.835720:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764673.835738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.835748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020dadc00 x1876702067871232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.835758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067871232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.835765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.835768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.835771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764673.835777:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764673.835794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.835952:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020daf800 x1876702067871360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.835963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067871360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.835970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.835972:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.835975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764673.835980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067871488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764673.836000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.836010:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020dae300 x1876702067871488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.836021:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067871488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.836029:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.836031:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.836035:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067871488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764673.836040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067871616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764673.836058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764673.836304:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764673/real 1789764673] req@ffff936020dad180 x1876702067871616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764673.836317:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067871616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764673.836323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764673.836326:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764673.836330:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067871616:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764674.601995:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764675.626115:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764676.650461:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764677.674761:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764678.698029:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764678.954045:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954056:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764678.954060:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954070:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954080:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954087:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764678.954122:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067871744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954155:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954159:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764678.954161:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954165:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954169:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954172:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954181:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764678.954193:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067871872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954202:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954205:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764678.954207:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954245:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954250:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954254:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954259:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764678.954269:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067872000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954278:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954281:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764678.954282:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954285:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954288:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954291:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954295:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764678.954304:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067872128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954311:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954314:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764678.954315:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954318:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954320:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954323:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954326:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764678.954333:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067872256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954340:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954343:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764678.954344:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954347:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954349:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954352:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954355:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764678.954362:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067872384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764678.954369:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764678.954371:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764678.954373:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764678.954376:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2146 00000100:00080000:3.0:1789764678.954379:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764678.954382:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764678.954386:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764678.954395:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067872512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764678.954631:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067871744:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764678.954687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.954721:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa5f80 x1876702067871744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.954738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067871744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.954747:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.954750:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.954754:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067871744:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764678.954760:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764678.954779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.954825:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa6680 x1876702067871872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.954838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067871872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.954846:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.954849:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.954852:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067871872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764678.954858:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764678.954878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.954889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa4e00 x1876702067872000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.954901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067872000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.954913:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.954916:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.954918:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764678.954923:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764678.954942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.954952:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa7480 x1876702067872128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.954962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067872128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.954969:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.954972:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.954975:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764678.954980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764678.954999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.955010:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa5500 x1876702067872256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.955020:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067872256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.955027:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.955030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.955033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764678.955039:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067872384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764678.955057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.955067:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa6a00 x1876702067872384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.955079:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067872384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.955087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.955090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.955093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067872384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764678.955098:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067872512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764678.955115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764678.955125:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764678/real 1789764678] req@ffff936010aa4700 x1876702067872512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764678.955136:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067872512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764678.955143:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764678.955146:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764678.955149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067872512:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764679.723127:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764680.745941:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764681.770034:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764682.794148:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764683.817962:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764684.074008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.074032:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764684.074036:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.074047:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.074071:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.074079:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764684.074140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067872640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.074194:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.074198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764684.074200:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.074204:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.074208:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764684.074358:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067872640:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764684.074419:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.074530:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa7100 x1876702067872640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.074584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067872640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.074593:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.074597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.074612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067872640:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764684.075044:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075054:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764684.075084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067872768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.075119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.075123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764684.075137:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.075142:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.075147:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.075150:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764684.075178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067872896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.075199:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.075203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764684.075205:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.075209:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.075222:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.075226:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764684.075270:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067873024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.075290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.075294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764684.075295:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.075299:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.075302:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.075305:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075310:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764684.075338:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067873152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.075348:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.075351:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764684.075352:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.075356:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.075359:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.075362:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075366:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764684.075374:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067873280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764684.075403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764684.075406:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764684.075408:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764684.075411:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2151 00000100:00080000:3.0:1789764684.075415:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764684.075428:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764684.075433:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764684.075442:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067873408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764684.075792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764684.075849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.075889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa5180 x1876702067872768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.075920:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067872768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.075939:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.075942:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.075954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764684.075961:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764684.075994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.076022:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa4380 x1876702067872896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.076047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067872896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.076071:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.076074:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.076077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067872896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764684.076095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764684.076131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.076142:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa4a80 x1876702067873024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.076180:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067873024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.076187:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.076190:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.076207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764684.076223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764684.076272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.076295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa5880 x1876702067873152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.076307:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067873152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.076315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.076318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.076333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764684.076339:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067873280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764684.076357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.076368:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa5c00 x1876702067873280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.076379:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067873280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.076387:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.076390:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.076393:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067873280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764684.076398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067873408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764684.076414:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764684.076423:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764684/real 1789764684] req@ffff936010aa4000 x1876702067873408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764684.076451:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067873408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764684.076469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764684.076472:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764684.076474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067873408:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764684.842243:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764685.866240:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764686.894591:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764687.915770:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764688.938005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764689.194263:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194272:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764689.194276:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194285:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194295:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194302:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764689.194339:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067873536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194367:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194371:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764689.194373:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194377:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194380:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194384:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194392:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764689.194404:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067873664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194418:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194422:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764689.194423:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194427:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194429:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194432:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194436:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764689.194444:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067873792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194452:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194455:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764689.194457:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194460:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194462:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194465:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194468:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764689.194474:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067873920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194482:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194484:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764689.194486:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194489:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194492:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194495:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194498:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764689.194504:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067874048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194512:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194514:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764689.194516:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194519:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194521:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194524:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194527:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764689.194533:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067874176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764689.194541:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764689.194543:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764689.194545:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764689.194548:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2157 00000100:00080000:0.0:1789764689.194551:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764689.194554:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764689.194557:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764689.194563:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067874304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764689.194642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067873536:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764689.194695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.194751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dac380 x1876702067873536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.194770:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067873536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.194779:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.194783:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.194786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067873536:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764689.194792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764689.194840:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.194853:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020daf100 x1876702067873664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.194865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067873664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.194872:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.194875:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.194878:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764689.194883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764689.194902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.194913:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dac700 x1876702067873792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.194924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067873792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.194936:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.194939:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.194942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764689.194948:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764689.194974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.194985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dafb80 x1876702067873920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.194995:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067873920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.195002:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.195005:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.195008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067873920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764689.195013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764689.195033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.195044:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dac000 x1876702067874048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.195054:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067874048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.195061:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.195063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.195066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764689.195071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067874176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764689.195102:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.195112:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dad880 x1876702067874176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.195121:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067874176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.195128:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.195131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.195134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067874176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764689.195139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067874304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764689.195156:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764689.195165:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764689/real 1789764689] req@ffff936020dad500 x1876702067874304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764689.195176:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067874304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764689.195182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764689.195185:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764689.195188:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067874304:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764689.965966:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764690.987094:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764692.010778:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764693.034591:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764694.058174:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764694.314549:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314557:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764694.314566:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314576:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314585:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314591:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764694.314632:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067874432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314655:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314659:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764694.314660:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314664:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314667:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314670:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314688:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764694.314699:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067874560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314707:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314710:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764694.314712:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314715:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314718:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314721:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314726:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764694.314736:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067874688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314744:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314747:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764694.314749:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314752:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314765:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314769:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314773:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764694.314783:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067874816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314792:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314795:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764694.314796:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314816:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314819:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314822:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314826:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764694.314836:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067874944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314844:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314847:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764694.314848:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314851:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314854:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314857:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314861:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764694.314869:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067875072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764694.314876:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764694.314884:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764694.314886:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764694.314888:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2162 00000100:00080000:0.0:1789764694.314892:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764694.314895:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764694.314900:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764694.314908:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067875200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764694.315425:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067874432:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764694.315484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020daf480 x1876702067874432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315551:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067874432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315560:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315564:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067874432:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764694.315574:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764694.315596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020dae680 x1876702067874560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067874560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315643:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764694.315649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764694.315669:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020daed80 x1876702067874688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067874688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315722:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764694.315728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764694.315749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315771:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020dad500 x1876702067874816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315782:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067874816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315790:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315793:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315796:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764694.315834:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764694.315853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315864:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020dad880 x1876702067874944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315874:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067874944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315883:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315886:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067874944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764694.315891:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764694.315909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315920:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020dac000 x1876702067875072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315930:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067875072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315936:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315939:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.315942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764694.315947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764694.315966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764694.315976:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764694/real 1789764694] req@ffff936020dafb80 x1876702067875200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764694.315987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067875200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764719 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764694.315994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764694.315997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764694.316000:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875200:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764695.082110:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764696.106699:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764697.129995:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764698.154529:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764699.178296:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764699.435896:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.435906:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764699.435910:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.435920:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.435929:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.435936:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764699.435971:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067875328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.435998:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436002:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764699.436003:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436007:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436010:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436013:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436022:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764699.436032:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067875456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.436045:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436048:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764699.436049:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436053:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436056:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436059:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436063:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764699.436072:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067875584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.436081:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436083:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764699.436085:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436088:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436091:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436094:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436097:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764699.436107:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067875712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.436116:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436118:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764699.436131:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436134:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436139:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436142:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436146:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764699.436156:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067875840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.436165:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436168:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764699.436169:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436173:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436175:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436178:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436182:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764699.436192:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067875968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764699.436199:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764699.436202:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764699.436203:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764699.436206:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2167 00000100:00080000:1.0:1789764699.436209:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764699.436212:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764699.436216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764699.436224:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067876096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764699.438902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067875328:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764699.438976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439021:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef6300 x1876702067875328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439040:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067875328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439049:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439053:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439056:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067875328:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764699.439063:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764699.439085:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439097:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef6d80 x1876702067875456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067875456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439132:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439135:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764699.439141:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764699.439162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439173:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef4e00 x1876702067875584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439184:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067875584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439196:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439199:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439202:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764699.439209:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764699.439228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439238:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef7100 x1876702067875712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439249:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067875712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439256:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439259:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439262:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764699.439268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764699.439285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef4380 x1876702067875840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067875840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439314:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067875840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764699.439326:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764699.439343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439354:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef4000 x1876702067875968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439365:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067875968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439372:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439375:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067875968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764699.439383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764699.439401:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764699.439411:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764699/real 1789764699] req@ffff936010ef5c00 x1876702067876096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764699.439421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067876096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764699.439428:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764699.439431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764699.439434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876096:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764700.202583:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764701.227338:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764702.250093:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764703.276481:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764704.299442:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764704.554322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764704.554335:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554345:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554356:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554363:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764704.554395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067876224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764704.554429:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554432:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554435:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554438:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554445:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764704.554454:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067876352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764704.554467:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554470:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554472:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554475:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554479:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764704.554486:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067876480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554497:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764704.554498:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554501:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554504:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554507:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554511:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764704.554517:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067876608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764704.554529:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554532:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554534:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554537:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554540:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764704.554548:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067876736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554554:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554557:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764704.554558:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554561:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554563:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554566:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554568:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764704.554575:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067876864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764704.554582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764704.554646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764704.554648:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764704.554651:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2172 00000100:00080000:1.0:1789764704.554655:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764704.554658:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764704.554662:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764704.554673:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067876992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764704.554879:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067876224:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764704.554938:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.554974:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef4a80 x1876702067876224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.554989:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067876224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.554998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555002:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555005:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067876224:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764704.555012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764704.555029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555039:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef5180 x1876702067876352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067876352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555059:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764704.555066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764704.555080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555089:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef4700 x1876702067876480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555100:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067876480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555110:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764704.555120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764704.555137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef5880 x1876702067876608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555160:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067876608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764704.555191:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764704.555210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555221:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef5c00 x1876702067876736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555231:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067876736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555240:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555242:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555246:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067876736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764704.555251:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764704.555267:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555277:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef4000 x1876702067876864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555286:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067876864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555293:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555296:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555299:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764704.555304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764704.555322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764704.555333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764704/real 1789764704] req@ffff936010ef4380 x1876702067876992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764704.555343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067876992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764704.555349:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764704.555352:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764704.555355:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067876992:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764705.322073:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764706.346338:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764707.371486:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764708.394247:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764709.418202:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764709.675474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.675493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764709.675499:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.675511:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.675526:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.675535:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764709.675796:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067877120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.675857:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.675862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764709.675864:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.675868:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.675873:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.675876:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.675888:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764709.675900:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067877248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.675909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.675913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764709.675914:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.675918:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.675921:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.675925:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.675929:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764709.675938:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067877376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.675947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.675950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764709.675951:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.675955:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.675959:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.675963:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.675967:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764709.675978:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067877504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.675987:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.675990:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764709.675992:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.675995:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.675998:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.676002:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.676006:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764709.676018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067877632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.676024:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.676028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764709.676030:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.676033:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.676102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.676108:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.676114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764709.676125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067877760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764709.676209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764709.676212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764709.676214:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764709.676218:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2177 00000100:00080000:1.0:1789764709.676222:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764709.676225:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764709.676230:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764709.676242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067877888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764709.676992:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067877120:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764709.677097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef7100 x1876702067877120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677185:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067877120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677390:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677396:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067877120:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764709.677410:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764709.677438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677452:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef4e00 x1876702067877248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067877248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677477:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677479:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677486:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764709.677492:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764709.677512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef6d80 x1876702067877376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067877376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677549:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677556:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764709.677572:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764709.677593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677604:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef6300 x1876702067877504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067877504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677621:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677624:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677627:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764709.677634:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764709.677664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677676:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef7b80 x1876702067877632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067877632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677706:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067877632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764709.677712:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067877760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764709.677733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677744:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef7480 x1876702067877760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677755:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067877760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677762:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677765:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067877760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764709.677774:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067877888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764709.677794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764709.677858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764709/real 1789764709] req@ffff936010ef7800 x1876702067877888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764709.677870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067877888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764709.677878:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764709.677881:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764709.677885:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067877888:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764710.442492:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764711.466624:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764712.491834:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764713.514268:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764714.544128:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764714.794381:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794392:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764714.794397:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794409:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794422:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794431:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764714.794478:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067878016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794521:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794532:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764714.794534:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794539:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794543:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794546:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794558:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764714.794571:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067878144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794581:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794584:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764714.794585:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794589:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794592:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794596:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794600:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764714.794610:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067878272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794617:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794620:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764714.794622:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794625:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794629:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794632:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794637:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764714.794646:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067878400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794655:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794658:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764714.794660:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794663:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794666:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794669:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794673:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764714.794682:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067878528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794690:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794693:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764714.794695:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794699:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794702:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794705:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794709:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764714.794718:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067878656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764714.794726:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764714.794740:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764714.794742:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764714.794745:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2182 00000100:00080000:1.0:1789764714.794748:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764714.794752:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764714.794756:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764714.794764:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067878784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764714.795172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067878016:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764714.795258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795307:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef6680 x1876702067878016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067878016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795341:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067878016:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764714.795352:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764714.795372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef5f80 x1876702067878144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795393:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067878144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795401:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764714.795411:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764714.795425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef6a00 x1876702067878272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067878272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795459:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764714.795468:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764714.795485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795493:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef5500 x1876702067878400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067878400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795523:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795533:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764714.795542:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764714.795562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795572:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef7800 x1876702067878528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067878528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795591:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067878528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764714.795603:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067878656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764714.795619:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795628:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef7480 x1876702067878656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067878656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795646:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795652:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067878656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764714.795657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067878784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764714.795674:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764714.795682:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764714/real 1789764714] req@ffff936010ef7b80 x1876702067878784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764714.795693:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067878784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764714.795700:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764714.795703:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764714.795706:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067878784:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764715.562030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764716.587100:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764717.610205:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764718.634902:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764719.658914:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764719.915504:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915519:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764719.915524:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915538:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915552:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915562:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764719.915607:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067878912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915647:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915651:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764719.915653:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915657:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915661:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915665:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915679:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764719.915692:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067879040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915702:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915706:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764719.915708:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915712:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915715:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915719:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915724:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764719.915734:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067879168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915743:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915747:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764719.915749:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915753:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915757:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915760:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915765:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764719.915775:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067879296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764719.915789:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915792:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915795:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915846:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915851:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764719.915861:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067879424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764719.915875:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915878:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915882:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915886:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915890:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764719.915898:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067879552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764719.915906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764719.915909:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764719.915911:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764719.915915:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2187 00000100:00080000:3.0:1789764719.915918:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764719.915922:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764719.915926:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764719.915937:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067879680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764719.916053:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067878912:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764719.916139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916193:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa7800 x1876702067878912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067878912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916230:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916235:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067878912:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764719.916242:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764719.916265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916277:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa7b80 x1876702067879040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916289:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067879040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916297:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916300:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916303:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764719.916309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764719.916330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916341:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa6300 x1876702067879168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067879168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916360:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764719.916371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764719.916392:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa4000 x1876702067879296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916415:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067879296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916423:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916426:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916429:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764719.916434:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764719.916454:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916464:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa5c00 x1876702067879424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067879424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916485:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916499:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764719.916505:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067879552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764719.916527:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916538:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa5880 x1876702067879552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067879552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916558:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916561:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916564:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067879552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764719.916570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067879680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764719.916591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764719.916603:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764719/real 1789764719] req@ffff936010aa4a80 x1876702067879680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764719.916615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067879680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764719.916623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764719.916626:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764719.916629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067879680:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764720.684611:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764721.706114:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764722.730993:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764723.754190:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764724.777984:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764725.034540:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764725.034553:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034563:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034574:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034595:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764725.034634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067879808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764725.034664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764725.034670:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034673:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034677:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034681:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764725.034689:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764725.034701:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067879936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764725.034709:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034712:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764725.034714:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034718:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034722:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034726:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764725.034732:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764725.034744:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067879808:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764725.034745:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067880064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764725.034760:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034763:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764725.034764:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034768:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034772:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034775:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764725.034779:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764725.034791:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067880192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764725.034829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764725.034844:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034848:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764725.034850:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034854:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034857:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034861:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764725.034867:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764725.034880:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067880320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764725.034888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034892:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764725.034893:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034897:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034900:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034903:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764725.034907:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764725.034916:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067880448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764725.034923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764725.034926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764725.034928:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764725.034931:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2192 00000100:00080000:1.0:1789764725.034934:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764725.034937:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764725.034940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef6300 x1876702067879808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789764725.034942:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764725.034955:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067880576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764725.034966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067879808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.034979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.034984:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.034988:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067879808:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764725.034995:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764725.035017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef6d80 x1876702067879936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035038:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067879936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035048:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.035050:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067879936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764725.035070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764725.035086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef4e00 x1876702067880064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067880064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.035117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764725.035123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764725.035139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035147:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef7100 x1876702067880192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067880192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035166:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.035169:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764725.035174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764725.035191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035202:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef4380 x1876702067880320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067880320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.035225:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764725.035230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067880448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764725.035250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef4000 x1876702067880448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035272:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067880448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035409:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.035412:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067880448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764725.035417:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067880576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764725.035439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764725.035971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764725/real 1789764725] req@ffff936010ef5c00 x1876702067880576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764725.035986:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067880576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764725.035994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764725.035997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764725.036000:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067880576:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764725.802277:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764726.827074:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764727.850022:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764728.874704:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764729.898160:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764730.153973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.153993:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764730.153997:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154007:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154018:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154025:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764730.154060:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067880704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764730.154099:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154103:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154106:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154119:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764730.154129:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067880832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154138:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154141:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764730.154143:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154146:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154149:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154153:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154157:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764730.154166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067880960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764730.154179:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154183:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154186:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154189:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154194:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764730.154203:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067881088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154212:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154215:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764730.154216:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154219:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154222:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154225:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764730.154238:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067881216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764730.154249:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154251:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154253:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154256:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154260:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764730.154269:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067881344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764730.154276:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764730.154278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764730.154279:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764730.154282:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2197 00000100:00080000:3.0:1789764730.154285:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764730.154288:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764730.154291:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764730.154299:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067881472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764730.155626:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067880704:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764730.155764:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764730.155824:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764730.155854:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764730.155881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764730.155903:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067881344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764730.155927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067881472:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764730.922123:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764731.946313:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764732.969997:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764733.226467:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764733.226506:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764733.226522:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226558:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226570:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226577:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226584:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226590:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764733.226597:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764733.226790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa4380 x1876702067880704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226833:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067880704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226851:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067880704:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764733.226858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa5180 x1876702067880832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226867:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067880832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226874:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226876:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226880:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764733.226885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa7100 x1876702067880960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226895:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067880960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226901:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226908:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067880960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764733.226914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa4700 x1876702067881088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067881088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226930:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226933:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226936:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764733.226942:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa6a00 x1876702067881216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226953:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067881216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226960:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764733.226971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa5500 x1876702067881344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.226981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067881344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.226987:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.226990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.226993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067881344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764733.226999:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764730/real 1789764733] req@ffff936010aa7480 x1876702067881472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764733.227009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067881472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764733.227017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764733.227024:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764733.227028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067881472:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764733.994113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764735.017983:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764735.273961:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.273971:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764735.273975:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.273993:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274004:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274013:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764735.274049:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067881600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274074:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274078:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764735.274080:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274084:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274088:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274091:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274100:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764735.274110:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067881728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274119:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274122:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764735.274123:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274127:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274130:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274133:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274137:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764735.274146:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067881856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274155:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274158:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764735.274160:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274163:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274166:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274169:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274173:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764735.274183:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067881984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274194:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764735.274195:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274198:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274202:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274205:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764735.274217:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067882112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274226:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274229:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764735.274231:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274234:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274237:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274240:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274245:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764735.274254:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067882240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764735.274262:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764735.274265:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764735.274267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764735.274270:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2203 00000100:00080000:0.0:1789764735.274273:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764735.274276:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764735.274280:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764735.274289:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067882368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764735.274758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067881600:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764735.274830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.274864:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020dac700 x1876702067881600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.274880:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067881600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.274890:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.274894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.274897:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067881600:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764735.274904:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764735.274924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.274934:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020daf100 x1876702067881728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.274945:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067881728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.274952:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.274956:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.274959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764735.274964:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764735.274979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.274989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020dac380 x1876702067881856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.274999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067881856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.275007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.275009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.275012:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764735.275016:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764735.275033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.275043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020dad180 x1876702067881984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.275053:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067881984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.275060:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.275063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.275066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067881984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764735.275071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764735.275089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.275098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020dae300 x1876702067882112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.275108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067882112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.275115:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.275118:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.275121:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764735.275126:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067882240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764735.275143:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.275152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020daf800 x1876702067882240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.275163:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067882240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.275170:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.275172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.275175:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067882240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764735.275180:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067882368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764735.275198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764735.275208:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764735/real 1789764735] req@ffff936020dadc00 x1876702067882368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764735.275226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067882368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764760 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764735.275233:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764735.275235:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764735.275238:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067882368:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764736.043123:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764737.066124:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764738.090187:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764739.115081:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764740.138212:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764740.394030:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764740.394040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764740.394045:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394068:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.394079:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.394086:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764740.394119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067882496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764740.394145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764740.394149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764740.394151:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394155:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.394158:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.394161:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764740.394171:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764740.394193:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067882496:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764740.394252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764740.394291:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067882624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764740.394295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef5880 x1876702067882496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764740.394313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764740.394318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764740.394320:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394324:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.394329:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.394334:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764740.394348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764740.394353:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067882496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764740.394364:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067882752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764740.394366:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.394374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.394382:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067882496:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764740.394385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764740.394390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764740.394393:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394398:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.394402:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.394406:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394410:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882624:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764740.394413:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789764740.394455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef4700 x1876702067882624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764740.394484:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067882624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764740.394488:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067882880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764740.394510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.394517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.394524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882624:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764740.394528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764740.394533:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882752:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764740.394534:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764740.394538:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394545:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.394550:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.394555:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764740.394561:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789764740.394561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394577:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef5180 x1876702067882752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764740.394579:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067883008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764740.394605:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789764740.394607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067882752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764740.394610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764740.394614:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.394623:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:2.0:1789764740.394626:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764740.394628:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764740.394632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789764740.394636:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882752:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764740.394644:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764740.394659:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067883136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764740.394660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764740.394688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef4a80 x1876702067882880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764740.394713:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067882880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764740.394720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.394724:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.394727:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067882880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764740.394733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764740.394751:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.394774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef5880 x1876702067883008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764740.394787:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067883008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764740.394795:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.395194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.395198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883008:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764740.395856:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764740.395860:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764740.395862:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764740.395867:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2208 00000100:00080000:1.0:1789764740.395873:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764740.395878:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764740.395884:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764740.395900:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067883264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764740.397130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067883136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764740.397191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.397213:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef5c00 x1876702067883136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764740.397229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067883136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764740.397239:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.397242:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.397246:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067883136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764740.397253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067883264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764740.397282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764740.397294:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764740/real 1789764740] req@ffff936010ef4000 x1876702067883264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764740.397304:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067883264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764765 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764740.397311:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764740.397314:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764740.397316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067883264:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764741.165734:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764742.187057:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764743.210003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764744.233992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764745.258219:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764745.514142:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514151:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764745.514155:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514165:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514174:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514181:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764745.514232:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067883392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514260:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514264:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764745.514265:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514269:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514272:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514284:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514293:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764745.514311:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067883520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514320:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514324:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764745.514325:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514328:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514331:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514334:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514337:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764745.514348:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067883648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514355:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514358:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764745.514359:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514362:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514364:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514367:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514371:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764745.514391:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067883776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514399:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514402:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764745.514404:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514407:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514410:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514413:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514417:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764745.514425:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067883904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514432:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514435:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764745.514437:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514444:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514447:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514450:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514454:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764745.514462:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067884032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764745.514469:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764745.514471:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764745.514473:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764745.514476:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2213 00000100:00080000:2.0:1789764745.514478:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764745.514481:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764745.514484:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764745.514491:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067884160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764745.514915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067883392:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764745.514970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.515007:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061dfb80 x1876702067883392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.515024:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067883392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.515034:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.515038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.515042:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067883392:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764745.515048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764745.515075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.515086:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061de680 x1876702067883520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.515097:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067883520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.515103:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.515107:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.515109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764745.515114:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764745.515144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.515155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061df800 x1876702067883648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.515166:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067883648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.515174:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.515177:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.515179:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764745.515184:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764745.515203:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.515212:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061dca80 x1876702067883776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.515223:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067883776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.516390:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.516396:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.516400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764745.516407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764745.516440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.516456:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061dd180 x1876702067883904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.516468:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067883904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.516476:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.516479:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.516482:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067883904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764745.516489:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764745.516508:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.516519:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061dce00 x1876702067884032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.516531:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067884032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.516539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.516542:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.516545:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764745.516551:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764745.517657:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764745.517679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764745/real 1789764745] req@ffff9360061dc000 x1876702067884160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764745.517695:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067884160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764745.517703:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764745.517706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764745.517709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884160:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764746.291404:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764747.307795:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764748.329993:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764749.356627:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764750.379522:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764750.634593:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634601:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764750.634605:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634613:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634622:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634629:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764750.634665:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067884288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634704:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634710:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764750.634712:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634716:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634721:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634725:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634733:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764750.634745:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067884416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634754:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634757:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764750.634759:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634763:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634766:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634769:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634773:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764750.634782:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067884544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634796:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634842:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764750.634844:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634847:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634851:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634854:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634859:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764750.634871:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067884672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634880:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634882:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764750.634884:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634887:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634890:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634893:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634897:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764750.634906:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067884800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634924:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634927:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764750.634929:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634932:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634936:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634941:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634946:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764750.634951:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067884288:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764750.634959:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067884928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764750.634969:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764750.634973:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764750.634975:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764750.634979:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2218 00000100:00080000:0.0:1789764750.634983:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764750.634986:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764750.634991:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764750.635002:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067885056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764750.635011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.635049:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020daea00 x1876702067884288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.635066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067884288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.635076:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.635081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.635086:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067884288:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764750.635106:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764750.635125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.635136:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020dadf80 x1876702067884416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.635148:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067884416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.635156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.635159:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.635162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764750.635168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764750.635188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.635201:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020dace00 x1876702067884544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.635213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067884544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.635221:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.635224:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.635227:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764750.635232:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764750.635252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.635262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020dadc00 x1876702067884672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.635707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067884672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.635715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.635718:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.635721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764750.635787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764750.636778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.636985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020daf800 x1876702067884800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.637003:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067884800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.637013:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.637017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.637021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067884800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764750.637029:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764750.637053:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.637064:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020dae300 x1876702067884928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.637076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067884928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.637083:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.637087:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.637090:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067884928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764750.637096:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764750.637115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764750.637126:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764750/real 1789764750] req@ffff936020dad180 x1876702067885056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764750.637137:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067885056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764750.637145:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764750.637148:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764750.637152:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885056:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764751.402557:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764752.426793:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764753.450363:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764754.475996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764755.499548:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764755.754386:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.754396:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764755.754400:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.754410:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.754429:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.754443:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764755.754479:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067885184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.754509:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.754513:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764755.754515:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.754519:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.754522:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.754526:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.754536:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764755.754547:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067885312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.754556:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.754559:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764755.754561:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.754564:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.754568:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.754571:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.754576:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764755.754586:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067885440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.754594:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.754597:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764755.754598:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.754614:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.754618:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.754622:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.754626:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764755.754828:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067885184:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764755.754884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.754921:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dac380 x1876702067885184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.754939:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067885184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.754949:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.754953:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.754956:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067885184:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764755.754963:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764755.754981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.754991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020daf100 x1876702067885312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755002:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067885312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755012:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755015:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764755.755020:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764755.755034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.755043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dac700 x1876702067885440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755054:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067885440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755060:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764755.755180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067885568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.755200:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.755203:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764755.755205:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.755209:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.755212:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.755216:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.755222:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764755.755244:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067885696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.755252:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.755255:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764755.755256:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.755260:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.755263:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.755266:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.755269:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764755.755277:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067885824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764755.755285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764755.755288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764755.755289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764755.755292:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2223 00000100:00080000:0.0:1789764755.755295:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764755.755298:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764755.755302:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764755.755309:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067885952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764755.755410:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764755.755446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.755457:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dafb80 x1876702067885568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067885568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755477:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755480:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764755.755488:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764755.755503:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.755511:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dac000 x1876702067885696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067885696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067885696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764755.755538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764755.755552:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.755559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dad880 x1876702067885824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755569:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067885824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755580:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755583:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764755.755588:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764755.755611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764755.755621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764755/real 1789764755] req@ffff936020dad500 x1876702067885952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764755.755631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067885952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764755.755887:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764755.755890:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764755.755894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067885952:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764756.525044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764757.548118:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764758.570087:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764759.596185:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764760.618048:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764760.874107:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874115:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764760.874119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874128:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874138:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874145:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764760.874178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067886080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764760.874216:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874220:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764760.874221:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874225:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874229:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789764760.874258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067886080:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764760.874275:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764760.874287:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764760.874302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067886208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764760.874313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:2.0:1789764760.874319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764760.874321:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874328:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874333:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874342:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764760.874347:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764760.874367:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067886336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764760.874373:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef4380 x1876702067886080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764760.874384:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874390:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764760.874392:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874397:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00100000:2.0:1789764760.874399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067886080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764760.874402:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874410:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789764760.874415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789764760.874417:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764760.874429:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067886464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764760.874438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764760.874444:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874447:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874450:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874453:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764760.874457:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764760.874465:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067886592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764760.874483:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874487:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764760.874489:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874492:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874495:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874500:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764760.874504:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764760.874514:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067886720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764760.874526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.874531:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067886080:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764760.874532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764760.874539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764760.874541:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764760.874546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2228 00000100:00080000:1.0:1789764760.874550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764760.874554:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764760.874558:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886208:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764760.874559:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764760.874577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067886848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764760.874590:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.874605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef7100 x1876702067886208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.874617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067886208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.874625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.874629:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.874632:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764760.875309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764760.875335:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.875348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef4e00 x1876702067886336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.875366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067886336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.875374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.875377:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.875380:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764760.875386:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764760.875403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.875412:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef6d80 x1876702067886464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.875422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067886464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.875436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.875438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.875441:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764760.875446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764760.875462:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.875471:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef6300 x1876702067886592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.875481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067886592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.875489:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.875492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.875495:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067886592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764760.875500:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067886720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764760.875514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.875533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef7b80 x1876702067886720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.875544:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067886720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.875551:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.875554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.875557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067886720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764760.875576:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067886848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764760.875593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764760.875602:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764760/real 1789764760] req@ffff936010ef7480 x1876702067886848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764760.875612:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067886848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764760.875620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764760.875622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764760.875625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067886848:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764761.644407:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764762.667446:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764763.690199:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764764.717223:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764765.745185:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764765.996268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996279:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764765.996283:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996293:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996303:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996309:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764765.996346:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067886976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764765.996384:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996388:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996392:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996395:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996404:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764765.996416:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067887104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996428:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764765.996430:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996433:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996436:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996440:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996444:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764765.996452:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067887232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996460:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996463:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764765.996464:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996467:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996469:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996472:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996476:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764765.996485:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067887360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996493:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996495:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764765.996497:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996501:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996504:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996507:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996512:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764765.996521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067887488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764765.996534:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996544:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996548:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764765.996558:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067887616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764765.996567:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764765.996570:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764765.996572:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764765.996575:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2233 00000100:00080000:1.0:1789764765.996579:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764765.996583:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764765.996587:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764765.996596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067886976:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764765.996599:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067887744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789764765.996658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.996693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef7800 x1876702067886976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.996711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067886976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.996721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.996725:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.996729:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067886976:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764765.996737:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764765.996759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.996772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef5500 x1876702067887104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.996786:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067887104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.996794:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.996842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.996846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764765.996853:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764765.996874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.996886:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef6a00 x1876702067887232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.996897:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067887232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.996905:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.996908:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.996911:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764765.996917:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764765.996935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.996945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef5f80 x1876702067887360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.996956:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067887360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.996963:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.996966:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.996969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764765.996974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764765.996990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.996998:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef6680 x1876702067887488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.997009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067887488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.997016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.997019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.997022:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067887488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764765.997027:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067887616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764765.997043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.997052:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef7480 x1876702067887616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.997063:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067887616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.997070:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.997073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.997075:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067887616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764765.997104:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067887744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764765.997122:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764765.997131:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764765/real 1789764765] req@ffff936010ef7b80 x1876702067887744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764765.997142:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067887744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764765.997149:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764765.997163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764765.997166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067887744:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764766.761964:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764767.786005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764768.810375:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764769.834081:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764770.858077:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764771.114139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114148:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764771.114153:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114162:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114171:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114178:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764771.114214:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067887872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114243:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114247:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764771.114249:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114253:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114256:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114260:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114268:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764771.114280:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067888000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114289:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114293:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764771.114294:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114297:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114301:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114303:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114308:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764771.114317:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067888128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114345:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114348:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764771.114350:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114353:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114356:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114359:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114363:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764771.114372:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067888256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114379:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114382:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764771.114383:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114386:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114388:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114392:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114395:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764771.114401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067888384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114408:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114411:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764771.114412:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114415:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114418:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114421:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114424:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764771.114430:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067888512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764771.114438:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764771.114440:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764771.114442:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764771.114445:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2238 00000100:00080000:0.0:1789764771.114447:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764771.114449:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764771.114452:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764771.114459:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067888640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789764771.114590:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067887872:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789764771.114643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114686:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020daed80 x1876702067887872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114702:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067887872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.114710:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.114713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.114716:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067887872:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789764771.114723:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764771.114741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020dae680 x1876702067888000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067888000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.114770:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.114772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.114775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764771.114780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764771.114795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114823:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020daf480 x1876702067888128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114834:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067888128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.114842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.114844:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.114847:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764771.114852:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764771.114870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114879:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020dad500 x1876702067888256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067888256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.114896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.114899:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.114902:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764771.114906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789764771.114923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020dad880 x1876702067888384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067888384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.114949:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.114951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.114954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789764771.114959:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067888512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764771.114975:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.114985:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020dac000 x1876702067888512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.114994:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067888512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.115001:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.115003:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.115006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067888512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789764771.115011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067888640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789764771.115026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789764771.115035:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764771/real 1789764771] req@ffff936020dafb80 x1876702067888640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764771.115045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067888640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764771.115051:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789764771.115054:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789764771.115057:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067888640:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764771.884194:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764772.906009:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764773.932165:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764774.956259:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764775.978119:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764776.233899:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764776.233910:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764776.233914:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764776.233935:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.233946:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.233954:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764776.236617:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067888768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764776.236715:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067888768:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764776.236720:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764776.236778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.236852:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa4e00 x1876702067888768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.236870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067888768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.236881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764776.236885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764776.236886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764776.236888:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764776.236889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067888768:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764776.237017:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.237023:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237030:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764776.237040:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764776.237054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067888896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764776.237145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764776.237150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764776.237152:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764776.237156:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.237160:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237165:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764776.237170:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764776.237182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067889024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764776.237192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764776.237195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764776.237197:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764776.237200:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.237203:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237207:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764776.237211:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764776.237221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067889152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764776.237230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764776.237232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764776.237234:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764776.237238:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.237241:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237244:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764776.237247:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764776.237255:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067889280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764776.237262:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764776.237265:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764776.237267:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764776.237270:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00080000:3.0:1789764776.237273:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237316:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764776.237343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789764776.237345:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764776.237358:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa6680 x1876702067888896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764776.237358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067889408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764776.237370:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067888896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764776.237374:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764776.237377:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764776.237377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764776.237378:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764776.237380:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764776.237382:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2244 00000100:00100000:0.0:1789764776.237383:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067888896:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764776.237385:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764776.237388:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889024:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789764776.237393:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764776.237401:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067889536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764776.237407:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237417:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa5f80 x1876702067889024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.237427:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067889024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.237435:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764776.237438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764776.237441:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764776.237446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764776.237461:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa4e00 x1876702067889152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.237475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067889152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.237480:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764776.237481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764776.237483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764776.237486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764776.237495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237501:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa7480 x1876702067889280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.237507:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067889280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.237511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764776.237513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764776.237514:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764776.237529:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067889408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764776.237539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237545:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa5500 x1876702067889408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.237551:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067889408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.237556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764776.237557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764776.237560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067889408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764776.237564:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067889536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764776.237580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764776.237591:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764776/real 1789764776] req@ffff936010aa6a00 x1876702067889536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764776.237601:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067889536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764801 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764776.237608:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764776.237611:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764776.237614:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067889536:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764777.002156:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764778.026776:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764779.050017:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764780.074037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764781.098320:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764781.353917:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.353926:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764781.353930:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764781.353939:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00080000:3.0:1789764781.353949:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.353956:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764781.353992:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067889664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764781.354024:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.354027:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764781.354029:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764781.354032:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00080000:3.0:1789764781.354036:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.354048:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.354057:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764781.354070:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067889792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764781.354080:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.354083:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764781.354085:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764781.354091:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00100000:0.0:1789764781.354093:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067889664:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764781.354095:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.354100:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.354105:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764781.354120:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067889920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764781.354135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.354139:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764781.354141:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789764781.354176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764781.354214:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00080000:3.0:1789764781.354219:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.354224:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.354229:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764781.354237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa4700 x1876702067889664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764781.354243:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067890048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764781.354253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.354259:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764781.354263:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764781.354269:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00080000:3.0:1789764781.354273:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.354280:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.354285:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764781.354292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067889664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764781.354296:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067890176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764781.354315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.354327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.354331:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067889664:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764781.354340:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764781.354506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764781.354511:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.354516:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764781.354518:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764781.354522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa7100 x1876702067889792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764781.354526:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00100000:0.0:1789764781.354538:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067889792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.354546:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.354549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.354553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764781.354578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764781.354596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764781.354605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa5180 x1876702067889920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764781.354616:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067889920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.354628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.354631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.354634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067889920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764781.354639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764781.354984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764781.354997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa4380 x1876702067890048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764781.355008:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067890048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.355016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.355019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.355021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764781.355028:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764781.355059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764781.355069:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa4a80 x1876702067890176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764781.355080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067890176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.355087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.355090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789764781.357289:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.357302:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.357316:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764781.357353:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067890304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764781.357373:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764781.357377:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764781.357381:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764781.357386:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2249 00000100:00080000:3.0:1789764781.357390:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764781.357393:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764781.357398:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764781.357411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067890432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764781.357600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764781.357643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067890304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764781.357698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764781.357734:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa5880 x1876702067890304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764781.357750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067890304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.357759:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.357763:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.357767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067890304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764781.357773:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067890432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764781.357817:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764781.357830:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764781/real 1789764781] req@ffff936010aa5c00 x1876702067890432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764781.357843:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067890432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764806 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764781.357851:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764781.357854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764781.357857:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067890432:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764782.124271:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764783.146084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764784.169968:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764785.193988:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764786.218050:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764786.474998:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475018:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764786.475022:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475032:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475043:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475050:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764786.475084:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067890560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475118:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475122:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764786.475124:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475128:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475131:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475135:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475144:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764786.475156:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067890688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475165:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475168:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764786.475170:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475174:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475178:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475182:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475186:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764786.475196:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067890816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475205:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475208:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764786.475210:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475214:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475217:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475220:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475224:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764786.475232:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067890944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475241:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475244:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764786.475246:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475249:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475252:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475255:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475259:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764786.475269:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067891072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475277:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475281:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764786.475283:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475286:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475289:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475292:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475296:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764786.475305:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067891200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764786.475314:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764786.475317:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764786.475319:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764786.475323:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2254 00000100:00080000:3.0:1789764786.475326:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764786.475330:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764786.475334:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764786.475343:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067891328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764786.475440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067890560:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764786.475495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa4000 x1876702067890560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067890560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475558:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067890560:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764786.475573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764786.475594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475606:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa6300 x1876702067890688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475618:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067890688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764786.475640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764786.475659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475669:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa7b80 x1876702067890816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067890816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475693:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475696:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764786.475701:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764786.475720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475731:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa7800 x1876702067890944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067890944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067890944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764786.475761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764786.475779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa5c00 x1876702067891072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067891072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475900:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764786.475906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067891200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764786.475925:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475936:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa5880 x1876702067891200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.475947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067891200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.475953:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.475956:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.475959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067891200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764786.475965:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067891328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764786.475984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764786.475994:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764786/real 1789764786] req@ffff936010aa4a80 x1876702067891328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764786.476018:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067891328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764786.476026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764786.476029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764786.476033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067891328:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764787.242025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764788.266034:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764789.290031:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764790.317703:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764791.342191:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764791.597608:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597619:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764791.597623:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.597633:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597644:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597651:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764791.597688:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067891456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764791.597725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597729:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764791.597731:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.597735:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597738:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597742:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764791.597750:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764791.597761:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067891584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764791.597771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597774:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764791.597775:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.597779:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597782:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597786:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764791.597790:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764791.597830:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067891712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764791.597837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067891456:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764791.597846:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597850:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764791.597852:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.597856:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597861:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597865:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764791.597870:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764791.597882:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067891840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764791.597891:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597894:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764791.597897:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789764791.597899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764791.597903:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597908:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597912:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764791.597917:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764791.597928:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067891968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764791.597937:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764791.597942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789764791.597943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa4380 x1876702067891456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764791.597944:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.597951:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.597958:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.597974:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764791.597979:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764791.597981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067891456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.597994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789764791.597994:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067892096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764791.598003:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598010:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067891456:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764791.598022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789764791.598024:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891584:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764791.598029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764791.598033:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764791.598038:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2259 00000100:00080000:3.0:1789764791.598042:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764791.598045:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789764791.598055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789764791.598061:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764791.598072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa5180 x1876702067891584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764791.598076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067892224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764791.598124:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067891584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.598133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.598137:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598140:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764791.598187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764791.598208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764791.598238:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa7100 x1876702067891712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764791.598262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067891712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.598286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.598289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764791.598296:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764791.598316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764791.598327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa4700 x1876702067891840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764791.598338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067891840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.598346:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.598349:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598352:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764791.598358:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764791.598374:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764791.598383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa6a00 x1876702067891968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764791.598393:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067891968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.598400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.598403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067891968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764791.598410:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764791.598426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764791.598435:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa5500 x1876702067892096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764791.598445:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067892096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.598452:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.598454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.598456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764791.600142:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764791.600187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764791.600206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764791/real 1789764791] req@ffff936010aa7480 x1876702067892224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764791.600221:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067892224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764791.600235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764791.601348:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764791.601353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892224:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764792.363295:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764793.385999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764794.410170:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764795.434134:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764796.461092:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764796.713877:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.713887:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764796.713891:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.713900:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.713910:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.713919:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764796.713966:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067892352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.713984:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.713987:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764796.713990:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.713993:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.713997:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.714000:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714009:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764796.714021:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067892480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.714030:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.714033:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764796.714035:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.714039:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.714042:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.714045:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714049:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764796.714059:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067892608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.714068:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.714071:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764796.714073:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.714076:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.714079:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.714083:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714087:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764796.714097:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067892736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.714107:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.714110:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764796.714112:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.714116:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.714120:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764796.714122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067892352:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789764796.714126:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714134:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764796.714145:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067892864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.714155:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.714158:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764796.714160:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.714163:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.714167:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.714170:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714174:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764796.714188:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067892992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764796.714196:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764796.714199:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764796.714200:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764796.714204:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2264 00000100:00080000:3.0:1789764796.714207:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764796.714210:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764796.714214:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764796.714223:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067893120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764796.714327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764796.714355:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764796.714376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764796.714399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764796.714417:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764796.714435:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067893120:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764797.482082:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764798.506752:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764799.531438:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789764799.786088:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:1.0:1789764799.786135:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789764799.786150:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786190:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786199:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786206:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786214:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786221:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:1.0:1789764799.786227:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789764799.788588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa5500 x1876702067892864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788625:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067892864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764799.788653:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa6a00 x1876702067892992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788666:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067892992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788684:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788687:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067892992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764799.788694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa4700 x1876702067893120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067893120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067893120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764799.788725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa4e00 x1876702067892352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788736:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067892352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788743:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788746:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788750:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067892352:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764799.788757:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa5f80 x1876702067892480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788781:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067892480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788793:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788796:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764799.788833:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa6680 x1876702067892608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788846:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067892608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788853:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788856:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764799.788866:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764796/real 1789764799] req@ffff936010aa7480 x1876702067892736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764799.788876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067892736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764799.788883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764799.788886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764799.788889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067892736:192.168.203.137@tcp:8:kworker.0 00010000:00010000:0.0:1789764800.555220:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764801.579316:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764801.834259:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.834266:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764801.834270:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.834279:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.834289:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.834296:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764801.834328:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067893248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.834360:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.834364:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764801.834366:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.834369:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.834374:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.834378:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.834387:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764801.834400:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067893376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.834410:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.834413:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764801.834415:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.834420:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.834423:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.834426:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.834431:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764801.834441:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067893504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.834450:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.834453:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764801.834455:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.834459:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.834463:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.834467:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.834948:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764801.834966:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067893632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.834978:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.834981:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764801.834982:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.834987:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.834993:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.834998:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.835002:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764801.835013:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067893760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.835022:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.835025:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764801.835026:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.835029:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.835032:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.835036:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.835040:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764801.835050:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067893888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764801.835058:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764801.835062:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764801.835063:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764801.835067:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2269 00000100:00080000:0.0:1789764801.835070:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764801.835073:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764801.835077:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764801.835085:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067894016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764801.835194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067893248:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764801.835263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.835309:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020dac700 x1876702067893248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.835326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067893248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.835336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.835341:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.835345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067893248:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764801.835354:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764801.835642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.835656:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020daf100 x1876702067893376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.835668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067893376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.835676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.835679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.835682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764801.835688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764801.835710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.835721:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020dac380 x1876702067893504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.835731:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067893504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.835739:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.835742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.835745:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764801.835751:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764801.835769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.835780:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020dad180 x1876702067893632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.835790:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067893632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.841575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.841586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.841593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764801.841607:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764801.841672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.841699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020dae300 x1876702067893760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.841723:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067893760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.841731:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.841734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.841737:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067893760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764801.841743:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067893888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764801.841766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.841779:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020daf800 x1876702067893888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.841792:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067893888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.841816:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.841819:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.841822:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067893888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764801.841829:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764801.841851:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764801.841863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764801/real 1789764801] req@ffff936020dadc00 x1876702067894016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764801.841874:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067894016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764801.841888:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764801.841900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764801.841903:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894016:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764802.602672:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764803.625984:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764804.650365:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764805.675296:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764806.698006:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764806.954849:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.954865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764806.954868:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.954878:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.954888:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.954895:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764806.954925:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067894144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.954954:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.954958:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764806.954960:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.954963:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.954966:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.954970:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.954978:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764806.954987:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067894272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.954997:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.955000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764806.955002:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.955006:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.955009:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.955012:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.955016:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764806.955025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067894400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.955034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.955037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764806.955038:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.955041:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.955044:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.955047:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.955051:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764806.955059:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067894528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.955067:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.955070:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764806.955072:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.955075:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.955078:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.955081:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.955084:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764806.955091:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067894656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.955099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.955102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764806.955103:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.955106:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.955109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.955113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.955116:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764806.955124:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067894784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764806.955132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764806.955134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764806.955136:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764806.955139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2274 00000100:00080000:3.0:1789764806.955141:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764806.955143:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764806.955146:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764806.955153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067894912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764806.955328:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067894144:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764806.955382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955426:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa7100 x1876702067894144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955441:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067894144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955457:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067894144:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764806.955464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764806.955493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955504:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa5180 x1876702067894272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955515:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067894272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955523:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955529:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764806.955535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764806.955550:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa4380 x1876702067894400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067894400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764806.955594:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764806.955609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955617:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa4a80 x1876702067894528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955628:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067894528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955636:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955639:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955642:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764806.955647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764806.955664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955674:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa5880 x1876702067894656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955685:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067894656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955704:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067894656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764806.955710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764806.955727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa5c00 x1876702067894784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955746:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067894784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955757:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764806.955764:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764806.955778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764806.955787:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764806/real 1789764806] req@ffff936010aa7800 x1876702067894912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764806.955909:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067894912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764806.955918:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764806.955922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764806.955926:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067894912:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764807.723617:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764808.747311:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764809.770667:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764810.795026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764811.818098:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764812.073974:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.073984:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764812.073988:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.073998:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074009:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074017:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764812.074054:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067895040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074092:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074096:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764812.074098:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074102:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074106:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074110:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074119:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764812.074130:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067895168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764812.074143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074147:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074150:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074153:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074158:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764812.074168:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067895296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074176:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764812.074181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074184:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074187:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074190:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764812.074206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067895424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074217:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764812.074218:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074221:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074224:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074228:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764812.074241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067895552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074249:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074252:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764812.074254:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074257:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074260:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074263:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074267:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764812.074275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067895680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764812.074283:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764812.074286:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764812.074287:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764812.074290:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2279 00000100:00080000:0.0:1789764812.074292:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764812.074296:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764812.074299:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764812.074308:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067895808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764812.074509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067895040:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764812.074581:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074629:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020dace00 x1876702067895040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067895040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.074657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.074661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.074664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067895040:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764812.074671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764812.074693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020dadf80 x1876702067895168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074717:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067895168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.074726:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.074729:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.074732:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764812.074738:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764812.074755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074765:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020daea00 x1876702067895296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074776:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067895296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.074789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.074792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.074796:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764812.074838:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764812.074859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074871:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020dadc00 x1876702067895424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074883:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067895424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.074889:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.074892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.074895:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764812.074901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764812.074918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074928:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020daf800 x1876702067895552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074939:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067895552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.074947:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.074950:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.074954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067895552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764812.074959:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067895680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764812.074979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.074989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020dae300 x1876702067895680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.074999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067895680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.075007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.075009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.075012:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067895680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764812.075018:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067895808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764812.075036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764812.075046:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764812/real 1789764812] req@ffff936020dad180 x1876702067895808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764812.075057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067895808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764812.075065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764812.075067:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764812.075069:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067895808:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764812.848044:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764813.866061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764814.889975:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764815.915055:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764816.941360:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764817.194395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194405:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764817.194409:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194418:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194427:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194434:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764817.194469:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067895936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194500:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764817.194506:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194510:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194513:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194517:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194525:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764817.194537:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067896064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194546:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194549:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764817.194551:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194554:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194557:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194560:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194563:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764817.194571:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067896192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764817.194584:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194588:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194642:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194646:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194651:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764817.194660:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067896320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194669:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194672:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764817.194673:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194677:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194679:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194682:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194686:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764817.194693:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067896448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194701:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194704:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764817.194706:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194709:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194712:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194715:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194719:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764817.194726:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067896576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764817.194733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764817.194736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764817.194737:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764817.194741:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2285 00000100:00080000:0.0:1789764817.194743:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764817.194746:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764817.194749:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764817.194756:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067896704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764817.194864:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067895936:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764817.194918:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195003:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dac380 x1876702067895936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195020:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067895936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195029:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195033:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195036:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067895936:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764817.195042:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764817.195066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195077:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020daf100 x1876702067896064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067896064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195095:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764817.195107:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764817.195125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195135:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dac700 x1876702067896192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067896192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764817.195172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764817.195191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195201:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dafb80 x1876702067896320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067896320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195220:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195225:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764817.195231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764817.195256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dac000 x1876702067896448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067896448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195284:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195287:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764817.195297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067896576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764817.195324:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195334:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dad880 x1876702067896576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195345:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067896576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195354:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067896576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764817.195362:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067896704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764817.195379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764817.195389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764817/real 1789764817] req@ffff936020dad500 x1876702067896704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764817.195399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067896704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764817.195406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764817.195408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764817.195411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067896704:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764817.965874:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764818.986139:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764820.010603:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764821.035219:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764822.058080:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764822.316816:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.316827:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764822.316831:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.316841:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.316851:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.316858:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764822.316899:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067896832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764822.316930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.316934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764822.316935:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.316939:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.316943:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.316947:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.316956:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764822.316969:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067896960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764822.316978:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.316981:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764822.316982:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.316986:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.316988:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.316991:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.316996:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764822.317005:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067897088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764822.317013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.317017:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764822.317019:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.317023:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.317026:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.317030:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.317034:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764822.317042:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067897216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764822.317050:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.317053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764822.317055:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.317058:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.317061:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.317064:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.317068:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764822.317077:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067897344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764822.317111:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067896832:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764822.317152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.317156:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764822.317159:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.317163:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.317167:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789764822.317172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764822.317174:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.317181:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764822.317195:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067897472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764822.317208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764822.317211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764822.317213:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764822.317216:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2290 00000100:00080000:0.0:1789764822.317220:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764822.317223:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764822.317227:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764822.317236:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067897600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764822.317270:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020daf480 x1876702067896832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067896832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067896832:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764822.317315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764822.317336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317346:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020dae680 x1876702067896960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317357:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067896960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067896960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764822.317374:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764822.317389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020daed80 x1876702067897088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067897088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317423:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317426:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317429:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764822.317434:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764822.317453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317463:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020dad500 x1876702067897216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317474:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067897216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764822.317698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764822.317719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317731:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020dad880 x1876702067897344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317752:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067897344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317759:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317766:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764822.317771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067897472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764822.317790:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317822:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020dac000 x1876702067897472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317834:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067897472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317848:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067897472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764822.317854:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067897600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764822.317873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764822.317885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764822/real 1789764822] req@ffff936020dafb80 x1876702067897600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764822.317895:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067897600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764847 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764822.317903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764822.317905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764822.317908:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067897600:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764823.083078:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764824.106407:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764825.130070:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764826.154026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764827.178047:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764827.434073:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.434082:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764827.434086:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.434096:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.434106:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.434113:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764827.434147:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067897728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.434620:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067897728:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764827.434676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764827.434710:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.434714:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764827.434716:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.434721:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.434725:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.434729:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.434738:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764827.434751:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067897856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.436082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa7b80 x1876702067897728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.436106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067897728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.436118:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.436122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.436126:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067897728:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764827.436171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764827.436210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764827.436225:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa6300 x1876702067897856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.436238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067897856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.436246:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.436249:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.436252:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897856:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764827.436592:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.436597:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764827.436599:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.436604:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.436609:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.436614:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.436623:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764827.436642:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067897984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.436698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764827.436735:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764827.436759:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.436763:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764827.436765:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.436769:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.436773:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.436777:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.436782:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764827.436806:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067898112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.436926:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa4000 x1876702067897984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.436945:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067897984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.436954:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.436959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.436963:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067897984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764827.436979:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764827.437001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764827.437013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa6300 x1876702067898112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.437031:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067898112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.437039:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.437042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.437045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898112:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764827.437108:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.437112:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764827.437114:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.437118:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.437123:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.437127:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.437133:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764827.437146:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067898240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.437301:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764827.437326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764827.437348:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.437480:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764827.437483:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.437487:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.437493:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.437497:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.437503:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764827.437516:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067898368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.437565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa7b80 x1876702067898240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.437577:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067898240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.437585:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.437588:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.437591:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764827.437605:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067898368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764827.437624:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764827.437636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa7800 x1876702067898368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.437647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067898368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.437660:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.437663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.437666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067898368:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764827.437718:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764827.437722:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764827.437724:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764827.437728:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2295 00000100:00080000:3.0:1789764827.437732:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764827.437735:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764827.437741:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764827.437752:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067898496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764827.438151:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067898496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764827.438299:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764827.438312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764827/real 1789764827] req@ffff936010aa5c00 x1876702067898496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764827.438324:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067898496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764827.438331:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764827.438334:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764827.438336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067898496:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764828.202036:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764829.226070:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764830.250031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764831.274024:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764832.301361:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764832.553964:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.553973:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764832.553977:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.553987:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.553997:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554004:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764832.554039:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067898624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554069:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554073:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764832.554074:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554078:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554081:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554085:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554094:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764832.554104:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067898752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554113:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554116:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764832.554118:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554122:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554125:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554128:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554133:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764832.554142:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067898880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554151:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554153:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764832.554155:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554158:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554161:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554165:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554169:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764832.554177:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067899008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554185:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554188:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764832.554190:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554192:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554195:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554198:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554202:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764832.554209:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067899136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554218:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554220:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764832.554222:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554225:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554228:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554232:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554236:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764832.554243:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067899264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764832.554252:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764832.554255:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764832.554257:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764832.554260:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2300 00000100:00080000:0.0:1789764832.554263:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764832.554266:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764832.554269:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764832.554278:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067899392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764832.554353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067898624:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764832.554409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.554460:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020dac700 x1876702067898624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.554477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067898624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.554487:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.554500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.554504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067898624:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764832.554512:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764832.554933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.554946:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020daf100 x1876702067898752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.554959:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067898752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.554968:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.554971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.554974:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764832.554981:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764832.555000:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.555011:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020dac380 x1876702067898880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.555023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067898880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.555030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.555033:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.555036:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067898880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764832.555041:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764832.555060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.555070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020dad180 x1876702067899008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.555082:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067899008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.555090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.555092:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.555095:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764832.555101:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764832.555116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.555125:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020dae300 x1876702067899136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.555136:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067899136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.555143:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.555146:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.555149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764832.555154:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067899264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764832.555168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.555176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020daf800 x1876702067899264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.555186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067899264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.555199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.555202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.555205:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067899264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764832.555210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067899392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764832.555224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764832.555232:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764832/real 1789764832] req@ffff936020dadc00 x1876702067899392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764832.555243:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067899392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764832.555250:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764832.555253:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764832.555256:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067899392:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764833.322025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764834.346641:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764835.370906:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764836.393934:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764837.420470:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764837.673902:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.673912:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764837.673917:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.673928:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.673939:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.673945:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764837.673982:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067899520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674013:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674017:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764837.674019:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674024:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674028:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674032:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674041:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764837.674054:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067899648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674064:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764837.674070:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674074:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674078:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674082:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674086:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764837.674096:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067899776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674106:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674109:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764837.674110:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674114:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674117:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674120:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674124:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764837.674133:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067899904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674142:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674144:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764837.674146:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674149:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674152:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674155:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674159:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764837.674168:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067900032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674176:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764837.674181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674184:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674187:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674191:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764837.674203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067900160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764837.674211:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764837.674213:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764837.674215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764837.674218:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2305 00000100:00080000:0.0:1789764837.674221:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764837.674224:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764837.674228:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764837.674235:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067900288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764837.674881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067899520:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764837.674982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020daea00 x1876702067899520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675037:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067899520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675258:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675263:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067899520:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764837.675276:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764837.675302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675547:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020dadf80 x1876702067899648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675563:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067899648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675572:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675575:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675578:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764837.675585:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764837.675609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020dace00 x1876702067899776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067899776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764837.675667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764837.675687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020dadc00 x1876702067899904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067899904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675722:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675725:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067899904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764837.675732:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764837.675750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675761:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020daf800 x1876702067900032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675772:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067900032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675783:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764837.675791:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067900160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764837.675834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675846:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020dae300 x1876702067900160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675859:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067900160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675867:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675871:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067900160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764837.675880:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067900288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764837.675898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764837.675908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764837/real 1789764837] req@ffff936020dad180 x1876702067900288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764837.675918:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067900288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764837.675926:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764837.675929:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764837.675931:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067900288:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764838.442155:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764839.466361:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764840.490115:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764841.516749:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764842.540823:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764842.798430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.798449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764842.798453:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.798463:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.798475:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.798481:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764842.798519:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067900416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764842.798632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067900416:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764842.798695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764842.798770:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.798775:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764842.798777:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.798781:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.798785:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.798789:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.798814:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764842.798841:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067900544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764842.799516:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef6300 x1876702067900416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.799536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067900416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.799546:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.799549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.799553:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067900416:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764842.799696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764842.799728:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764842.799741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef6d80 x1876702067900544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.799754:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067900544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.799761:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.799764:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.799767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900544:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764842.800167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.800172:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764842.800174:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.800179:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.800184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.800190:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.800197:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764842.800217:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067900672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764842.800263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764842.800289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764842.800312:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.800315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764842.800317:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.800321:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.800325:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.800329:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.800334:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764842.800346:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067900800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764842.800388:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef4e00 x1876702067900672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.800399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067900672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.800406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.800409:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.800412:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764842.800424:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900800:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764842.800520:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.800524:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764842.800526:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.800531:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.800535:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.800539:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.800544:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764842.800556:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067900928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764842.800565:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789764842.800696:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764842.800712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef7100 x1876702067900800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.800725:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067900800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.800733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.800736:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.800739:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764842.800752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764842.800774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764842.800785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef4380 x1876702067900928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.800795:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067900928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764842.801065:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764842.801067:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.801071:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.801075:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.801080:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.801086:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764842.801098:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067901056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764842.801114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764842.801117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764842.801118:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764842.801122:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2310 00000100:00080000:1.0:1789764842.801125:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764842.801128:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764842.801132:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764842.801141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067901184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764842.801203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.801207:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.801210:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067900928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764842.801223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764842.801249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764842.801260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef4000 x1876702067901056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.801272:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067901056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.801280:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.801283:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.801286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764842.801291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764842.801311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764842.801321:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764842/real 1789764842] req@ffff936010ef5c00 x1876702067901184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764842.801332:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067901184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764842.801339:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764842.801342:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764842.801345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901184:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764843.562038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764844.587061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764845.609995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764846.634005:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764847.658151:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764847.914286:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.914296:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764847.914300:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.914308:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.914318:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.914324:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764847.914358:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067901312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764847.914657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067901312:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789764847.914711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764847.914901:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.914906:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764847.914908:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.914912:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.914916:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.914920:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.914929:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764847.914943:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067901440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764847.915126:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa5880 x1876702067901312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.915145:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067901312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.915154:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.915158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.915162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067901312:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764847.915184:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764847.915207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.915219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa4a80 x1876702067901440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.915231:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067901440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.915240:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.915243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.915247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901440:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764847.916351:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.916358:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764847.916360:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.916366:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.916372:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.916377:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.916385:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764847.916405:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067901568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764847.916595:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.916599:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764847.916600:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.916605:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.916609:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.916612:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.916618:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764847.916629:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067901696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764847.916639:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.916642:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764847.916644:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.916647:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.916650:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.916653:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.916657:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764847.916665:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067901824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764847.916674:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.916676:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764847.916678:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.916681:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.916684:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.916687:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.916691:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764847.916698:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067901952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764847.916707:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764847.916710:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764847.916712:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764847.916715:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2315 00000100:00080000:3.0:1789764847.916718:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764847.916721:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764847.916725:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764847.916732:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067902080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764847.916795:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764847.916843:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.916861:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa4380 x1876702067901568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.916875:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067901568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.916884:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.916887:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.916891:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764847.916898:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764847.916915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.916926:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa5180 x1876702067901696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.916937:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067901696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.916950:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.916953:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.916956:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764847.916961:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764847.916978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.916989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa7100 x1876702067901824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.916998:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067901824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.917004:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.917007:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.917009:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067901824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764847.917014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764847.917030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.917039:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa7480 x1876702067901952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.917049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067901952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.917055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.917058:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.917060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067901952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764847.917065:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764847.917080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764847.917090:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764847/real 1789764847] req@ffff936010aa6680 x1876702067902080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764847.917099:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067902080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764847.917106:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764847.917109:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764847.917111:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902080:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764848.681989:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764849.708428:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764850.730096:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764851.754259:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764852.778491:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764853.035621:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.035634:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764853.035639:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.035651:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.035665:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.035674:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764853.035723:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067902208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764853.035758:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.035762:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764853.035764:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.035768:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.035772:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.035776:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764853.035787:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764853.035836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067902208:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764853.035852:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067902336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764853.035864:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.035867:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764853.035869:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.035873:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.035877:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.035881:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764853.035887:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764853.035900:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067902464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764853.035909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764853.035914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764853.035915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764853.035920:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.035925:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.035929:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.035933:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764853.035939:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764853.035951:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067902592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764853.035960:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.035963:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764853.035964:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764853.035969:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef5880 x1876702067902208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764853.035971:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.035977:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.035984:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764853.035991:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764853.035999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067902208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764853.036005:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067902720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764853.036016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.036022:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.036029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067902208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789764853.036035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.036041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764853.036043:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.036047:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2320 00000100:00080000:1.0:1789764853.036051:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.036055:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764853.036062:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902336:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789764853.036062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764853.036081:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067902848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764853.036092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764853.036096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764853.036097:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764853.036101:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2320 00000800:00000100:0.0:1789764853.038947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764853.042292:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764853.042314:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764853.042333:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764853.042335:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef4a80 x1876702067902336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.042361:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067902336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.042374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.042379:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.042383:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764853.042396:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764853.042513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764853.042539:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef5180 x1876702067902464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.042551:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067902464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.042561:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.042564:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.042567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764853.042574:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764853.042607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764853.042618:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef4700 x1876702067902592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.042630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067902592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.042639:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.042642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.042645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764853.042651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789764853.042668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764853.042677:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef5c00 x1876702067902720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.042687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067902720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.042696:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.042699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.042702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067902720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764853.042732:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764853.042750:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764853.042759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef4000 x1876702067902848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.042770:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067902848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.042784:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.042788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.042791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764853.046313:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067902976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764853.050077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789764853.050179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764853.050220:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764853/real 1789764853] req@ffff936010ef4380 x1876702067902976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764853.050242:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067902976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764853.050255:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764853.050261:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789764853.050265:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067902976:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764853.803827:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764854.826133:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764855.849972:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764856.874565:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764857.898134:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764858.153932:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.153942:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764858.153946:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.153956:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.153976:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.153984:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764858.154016:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067903104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154045:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764858.154049:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154052:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154055:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154058:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154066:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764858.154076:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067903232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154092:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764858.154095:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154099:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154102:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154105:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154109:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764858.154117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067903360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764858.154128:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154131:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154133:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154136:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764858.154148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067903488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764858.154160:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154162:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154166:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154169:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154172:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764858.154180:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067903616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764858.154192:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154196:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154198:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154201:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154205:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764858.154215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067903744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764858.154222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764858.154225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764858.154226:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764858.154229:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2325 00000100:00080000:1.0:1789764858.154232:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764858.154235:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764858.154239:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764858.154247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067903872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764858.154400:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067903104:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764858.154460:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.154527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef7100 x1876702067903104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.154543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067903104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.154553:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.154569:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.154574:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067903104:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764858.154581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764858.154612:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.154624:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef4e00 x1876702067903232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.154636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067903232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.154649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.154653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.154656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764858.154662:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764858.154859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.154875:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef6d80 x1876702067903360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.154887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067903360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.154895:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.154898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.154902:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764858.154907:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764858.154929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.154942:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef6300 x1876702067903488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.154954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067903488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.154962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.154975:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.154980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764858.154986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764858.155005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.155016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef7b80 x1876702067903616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.155028:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067903616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.155035:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.155039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.155042:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067903616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764858.155047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067903744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764858.155066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.155077:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef7480 x1876702067903744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.155089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067903744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.155096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.155099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.155102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067903744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764858.155108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067903872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764858.155127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764858.155138:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764858/real 1789764858] req@ffff936010ef6680 x1876702067903872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764858.155149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067903872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764858.155156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764858.155158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764858.155162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067903872:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764858.924677:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764859.946018:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764860.970025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764861.994579:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764863.018382:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764863.273936:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.273952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764863.273957:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.273972:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.273987:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.273998:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764863.274065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067904000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274102:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274107:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764863.274109:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274112:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274116:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274121:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274132:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764863.274144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067904128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274157:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764863.274159:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274163:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274166:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274171:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274175:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764863.274186:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067904256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764863.274200:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274204:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274207:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274211:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274216:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764863.274225:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067904384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764863.274239:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274241:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274244:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274247:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764863.274262:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067904512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274271:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764863.274278:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274282:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274286:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274291:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274296:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764863.274308:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067904640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764863.274318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764863.274322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764863.274324:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764863.274328:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2331 00000100:00080000:1.0:1789764863.274332:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764863.274337:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764863.274348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764863.274359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067904768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764863.274740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067904000:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764863.274930:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764863.274968:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764863.274991:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764863.275012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764863.275079:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067904640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764863.275110:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067904768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764864.050883:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764865.068998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764866.091560:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764866.346072:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764866.346113:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764866.346128:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346167:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346175:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346182:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346190:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346197:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764866.346204:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764866.347747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef5f80 x1876702067904000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067904000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347794:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067904000:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764866.347827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef6a00 x1876702067904128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347839:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067904128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347847:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347851:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347854:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764866.347860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef5500 x1876702067904256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347881:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067904256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347888:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764866.347901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef7800 x1876702067904384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067904384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347925:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347928:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347931:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764866.347938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef6680 x1876702067904512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347949:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067904512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347955:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347958:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347961:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067904512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764866.347967:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef7480 x1876702067904640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.347978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067904640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.347985:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.347987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.347990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067904640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764866.347996:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764863/real 1789764866] req@ffff936010ef7b80 x1876702067904768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764866.348006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067904768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764888 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764866.348012:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764866.348016:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764866.348019:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067904768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764867.121636:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764868.138071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764868.396157:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396166:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764868.396169:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396178:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396188:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396195:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764868.396227:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067904896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396251:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396255:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764868.396256:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396260:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396263:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396267:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396276:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764868.396287:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067905024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396297:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396300:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764868.396301:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396305:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396309:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396312:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396316:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764868.396339:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067905152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396349:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396352:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764868.396354:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396357:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396360:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396364:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396368:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764868.396377:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067905280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396386:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396389:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764868.396391:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396394:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396397:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396400:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396412:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764868.396422:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067905408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396431:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396434:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764868.396435:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396438:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396441:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396444:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396448:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764868.396458:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067905536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764868.396466:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764868.396469:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764868.396481:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764868.396485:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2336 00000100:00080000:1.0:1789764868.396488:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764868.396492:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764868.396495:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764868.396505:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067905664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764868.396825:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067904896:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764868.396883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.396939:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef6300 x1876702067904896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.396955:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067904896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.396965:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.396969:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.396973:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067904896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764868.396980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764868.397002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef6d80 x1876702067905024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067905024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397035:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397041:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764868.397046:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764868.397065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397089:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef4e00 x1876702067905152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067905152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397110:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764868.397123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764868.397144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef7100 x1876702067905280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397165:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067905280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397172:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397175:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397178:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764868.397183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764868.397200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397210:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef4380 x1876702067905408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067905408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397233:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397236:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397239:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764868.397244:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067905536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764868.397264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef4000 x1876702067905536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397286:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067905536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397293:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397296:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397299:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067905536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764868.397305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067905664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764868.397699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764868.397745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764868/real 1789764868] req@ffff936010ef5c00 x1876702067905664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764868.397760:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067905664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764893 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764868.397776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764868.397779:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764868.397783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067905664:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764869.163793:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764870.186034:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764871.210021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764872.233981:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764873.258516:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764873.513950:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.513960:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764873.513964:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.513974:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.513984:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.513990:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764873.514027:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067905792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514057:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514060:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764873.514062:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514066:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514070:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514074:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514083:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764873.514093:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067905920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514103:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514105:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764873.514107:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514111:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514114:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514117:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514121:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764873.514130:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067906048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764873.514143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514146:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514149:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514152:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764873.514165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067906176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514175:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764873.514176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514179:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514182:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514185:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514189:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764873.514199:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067906304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514208:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514211:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764873.514213:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514216:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514219:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514223:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514227:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764873.514238:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067906432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764873.514247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764873.514250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764873.514251:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764873.514254:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2341 00000100:00080000:1.0:1789764873.514258:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764873.514261:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764873.514265:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764873.514276:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067906560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764873.514384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067905792:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764873.514439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.514741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef4700 x1876702067905792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.514759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067905792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.514770:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.514773:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.514776:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067905792:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764873.514784:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764873.514886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.514899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef5180 x1876702067905920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.516128:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067905920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.516139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.516143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.516147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067905920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764873.516156:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764873.517033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.517062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef4a80 x1876702067906048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.517077:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067906048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.517087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.517091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.517094:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764873.517102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764873.517132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.517144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef5880 x1876702067906176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.517155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067906176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.517162:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.517165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.517169:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764873.517174:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764873.517192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.517203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef5c00 x1876702067906304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.517230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067906304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.517237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.517240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.517243:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764873.517249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067906432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764873.517269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.517280:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef4000 x1876702067906432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.517291:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067906432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.517298:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.517301:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.517304:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067906432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764873.517309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067906560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764873.517328:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764873.517337:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764873/real 1789764873] req@ffff936010ef4380 x1876702067906560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764873.517347:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067906560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764873.517357:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764873.517361:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764873.517364:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067906560:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764874.283196:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764875.307322:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764876.333193:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764877.354187:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764878.378015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764878.633927:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.633937:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764878.633941:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.633949:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.633959:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.633966:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764878.634000:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067906688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634029:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634032:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764878.634034:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634045:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634049:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634052:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634060:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764878.634079:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067906816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634088:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634092:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764878.634093:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634097:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634100:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634103:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634107:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764878.634117:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067906944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634126:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634128:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764878.634130:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634133:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634135:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634138:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634142:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764878.634151:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067907072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634159:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634161:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764878.634162:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634165:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634168:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634171:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634175:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764878.634183:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067907200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634194:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764878.634195:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634198:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634200:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634203:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634207:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764878.634217:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067907328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764878.634224:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764878.634226:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764878.634227:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764878.634230:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2346 00000100:00080000:1.0:1789764878.634233:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764878.634236:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764878.634240:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764878.634247:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067907456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764878.634562:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067906688:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764878.634631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.634687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef7100 x1876702067906688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.634703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067906688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.634713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.634716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.634719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067906688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764878.634727:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764878.634759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.634770:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef4e00 x1876702067906816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.634780:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067906816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.634787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.634790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.634793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764878.634824:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764878.634844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.634855:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef6d80 x1876702067906944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.634864:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067906944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.634871:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.634873:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.634876:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067906944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764878.634881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764878.634899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.634908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef6300 x1876702067907072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.634916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067907072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.634923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.634926:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.634928:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764878.634933:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764878.634950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.634959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef7b80 x1876702067907200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.634969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067907200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.634976:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.634978:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.634981:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764878.634986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067907328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764878.635004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.635014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef7480 x1876702067907328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.635025:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067907328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.635031:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.635034:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.635037:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067907328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764878.635041:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067907456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764878.635058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764878.635068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764878/real 1789764878] req@ffff936010ef6680 x1876702067907456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764878.635076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067907456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764878.635088:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764878.635090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764878.635093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067907456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764879.402088:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764880.426004:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764881.454946:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764882.474069:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764883.501433:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764883.753934:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.753945:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764883.753948:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.753967:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.753977:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.753984:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764883.754021:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067907584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.754104:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067907584:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764883.754164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764883.754316:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.754320:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764883.754322:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.754326:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.754330:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.754334:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.754343:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764883.754357:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067907712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.754530:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef7800 x1876702067907584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.754549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067907584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.754558:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.754561:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.754565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067907584:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764883.754590:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764883.754613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764883.754624:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef5500 x1876702067907712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.754634:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067907712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.754641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.754644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.754647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907712:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764883.754739:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.754743:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764883.754745:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.754749:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.754754:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.754758:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.754763:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764883.754777:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067907840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.754883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764883.754908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764883.754948:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.754961:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764883.754963:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.754967:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.754971:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.754974:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.754979:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764883.754990:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067907968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.755082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef6a00 x1876702067907840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.755096:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067907840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.755104:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.755107:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.755110:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764883.755123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764883.755145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764883.755157:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef5f80 x1876702067907968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.755168:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067907968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.755176:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.755179:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.755182:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067907968:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789764883.755440:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.755444:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764883.755446:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.755451:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.755455:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.755459:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.755465:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764883.755477:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067908096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.755522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764883.755546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789764883.755568:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.755573:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764883.755575:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.755580:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.755584:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.755588:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.755594:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764883.755604:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067908224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.755640:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef5f80 x1876702067908096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.755653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067908096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.755660:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.755662:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.755665:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764883.755677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067908224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764883.755697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764883.755709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef6a00 x1876702067908224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.755721:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067908224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.755728:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.755731:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.755735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067908224:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789764883.756035:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764883.756039:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764883.756041:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764883.756045:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2351 00000100:00080000:1.0:1789764883.756050:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764883.756053:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764883.756059:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764883.756126:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067908352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764883.756201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067908352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764883.756228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764883.756240:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764883/real 1789764883] req@ffff936010ef5500 x1876702067908352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764883.756252:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067908352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764883.756265:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764883.756269:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764883.756272:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067908352:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764884.522049:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764885.546661:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764886.570040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764887.597310:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764888.618061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764888.874064:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.874073:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764888.874077:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.874086:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.874097:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.874104:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764888.874138:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067908480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.874170:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.874174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764888.874175:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.874179:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.874182:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.874186:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.874195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764888.874207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067908608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.874217:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.874220:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764888.874222:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.874239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.874243:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.874246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.874251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764888.874270:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067908736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.874279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.874282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764888.874284:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.874287:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.874290:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.874294:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.874299:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764888.874301:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067908480:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764888.874314:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067908864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.874325:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.874328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764888.874330:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.874333:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2356 00000800:00000100:1.0:1789764888.874362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764888.875809:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.875823:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.875833:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764888.875857:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067908992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.875870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.875874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764888.875877:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.875882:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.875886:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.875890:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.875895:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764888.875904:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067909120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764888.875912:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764888.875915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764888.875917:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764888.875920:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2356 00000100:00080000:0.0:1789764888.875924:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764888.875927:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764888.875931:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764888.875940:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067909248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764888.876631:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dac380 x1876702067908480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.876650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067908480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.876666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.876670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.876674:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067908480:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764888.876685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764888.876729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.876743:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020daf100 x1876702067908608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.876753:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067908608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.876761:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.876910:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.876916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764888.876923:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764888.876945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.876957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dac700 x1876702067908736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.876969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067908736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.876978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.876981:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.876985:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764888.877012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764888.877033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.877043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dafb80 x1876702067908864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.877052:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067908864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.877060:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.877063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.877066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764888.877071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764888.877088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.877098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dac000 x1876702067908992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.877108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067908992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.877114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.877117:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.877120:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067908992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764888.877124:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067909120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764888.877141:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.877151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dad880 x1876702067909120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.877161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067909120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.877168:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.877171:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.877174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067909120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764888.877179:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067909248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764888.877196:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764888.877206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764888/real 1789764888] req@ffff936020dad500 x1876702067909248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764888.877217:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067909248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764888.877224:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764888.877227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764888.877230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067909248:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764889.642262:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764890.666290:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764891.691004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764892.715301:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764893.740285:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764893.994083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994092:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764893.994096:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994106:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994116:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994122:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764893.994156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067909376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764893.994193:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994199:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994202:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994210:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764893.994229:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067909504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994238:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994241:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764893.994243:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994246:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994249:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994252:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994256:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764893.994264:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067909632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994273:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764893.994277:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994280:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994283:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994286:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994290:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764893.994296:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067909760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764893.994307:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994310:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994312:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994315:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994319:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764893.994325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067909888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764893.994336:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994339:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994341:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994344:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994347:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764893.994353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067910016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764893.994361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764893.994364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764893.994366:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764893.994369:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2361 00000100:00080000:1.0:1789764893.994371:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764893.994373:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764893.994376:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764893.994383:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067910144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764893.994464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067909376:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764893.994517:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994552:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef7800 x1876702067909376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994569:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067909376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994578:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067909376:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764893.994591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764893.994610:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef6680 x1876702067909504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067909504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994638:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994644:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764893.994649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764893.994663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef7480 x1876702067909632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067909632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994694:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764893.994699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764893.994713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef7b80 x1876702067909760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067909760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764893.994746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764893.994759:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef6300 x1876702067909888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994777:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067909888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994784:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994786:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994789:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067909888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764893.994793:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764893.994946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.994957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef6d80 x1876702067910016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.994968:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067910016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.994975:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.994978:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.994981:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764893.994987:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764893.995001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764893.995009:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764893/real 1789764893] req@ffff936010ef4e00 x1876702067910144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764893.995019:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067910144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764893.995032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764893.995035:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764893.995038:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910144:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764894.762048:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764895.786143:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764896.809993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764897.835814:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764898.862590:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764899.114292:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114307:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764899.114312:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114326:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114339:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114347:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764899.114397:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067910272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114431:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114435:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764899.114437:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114441:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114445:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114449:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114460:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764899.114472:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067910400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114482:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114485:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764899.114488:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114492:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114495:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114498:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114503:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764899.114512:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067910528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114521:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114524:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764899.114526:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114530:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114534:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114538:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114543:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764899.114553:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067910656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114562:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114564:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764899.114566:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114570:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114573:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114576:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114580:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764899.114589:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067910784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114598:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114601:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764899.114603:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114607:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114610:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114613:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114617:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764899.114629:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067910912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764899.114637:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764899.114640:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764899.114642:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764899.114646:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2366 00000100:00080000:3.0:1789764899.114649:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764899.114652:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764899.114655:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764899.114665:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067911040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764899.114873:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067910272:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764899.114953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa5f80 x1876702067910272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115020:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067910272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067910272:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764899.115048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764899.115069:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115081:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa4e00 x1876702067910400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115092:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067910400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115100:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115103:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115107:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764899.115113:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764899.115130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa4700 x1876702067910528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067910528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115162:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764899.115171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764899.115202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115212:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa6a00 x1876702067910656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115223:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067910656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115232:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115234:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764899.115243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764899.115321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa5500 x1876702067910784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115345:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067910784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115352:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115359:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067910784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764899.115365:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764899.115383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115395:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa6680 x1876702067910912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067910912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115417:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115421:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067910912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764899.115426:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764899.115443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764899.115453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764899/real 1789764899] req@ffff936010aa7480 x1876702067911040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764899.115465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067911040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764899.115472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764899.115475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764899.115478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911040:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764899.883338:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764900.906208:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764901.930477:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764902.953997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764903.981551:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764904.234241:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764904.234265:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234274:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234283:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234290:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764904.234324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067911168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764904.234358:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234362:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234365:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234368:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234377:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764904.234387:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067911296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234395:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764904.234400:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234404:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234410:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234413:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764904.234422:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067911424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764904.234434:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234438:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234440:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234444:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234447:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764904.234457:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067911552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764904.234470:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234473:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234476:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234479:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234483:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764904.234490:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067911680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764904.234503:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234506:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234509:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234512:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234515:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764904.234533:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067911808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764904.234541:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764904.234544:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764904.234546:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764904.234549:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2372 00000100:00080000:0.0:1789764904.234552:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764904.234555:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764904.234558:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764904.234567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067911936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764904.234710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067911168:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764904.234765:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.234834:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020daed80 x1876702067911168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.234849:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067911168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.234859:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.234862:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.234866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067911168:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764904.234873:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764904.234892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.234904:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020dae680 x1876702067911296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.234914:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067911296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.234923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.234926:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.234929:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764904.234934:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764904.234950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.234960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020daf480 x1876702067911424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.234970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067911424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.234978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.234981:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.234984:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764904.234989:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764904.235005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.235014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020dad500 x1876702067911552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.235030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067911552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.235038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.235040:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.235043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764904.235048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764904.235064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.235074:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020dad880 x1876702067911680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.235083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067911680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.235090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.235094:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.235097:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067911680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764904.235102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764904.235116:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.235126:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020dac000 x1876702067911808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.235155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067911808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.235168:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.235171:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.235174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764904.235178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764904.235195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764904.235205:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764904/real 1789764904] req@ffff936020dafb80 x1876702067911936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764904.235216:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067911936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764929 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764904.235223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764904.235225:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764904.235228:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067911936:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764905.007271:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764906.026067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764907.051033:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764908.075097:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764909.098194:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764909.355847:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.355854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764909.355858:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.355866:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.355883:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.355890:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789764909.355921:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067912064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.355948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.355952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764909.355954:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.355958:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.355961:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.355964:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.355974:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764909.355985:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067912192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.355993:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.355996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764909.355997:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.356000:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.356003:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.356006:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.356010:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764909.356018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067912320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.356025:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.356028:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764909.356029:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.356032:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.356035:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.356039:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.356042:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764909.356050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067912448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.356057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.356060:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789764909.356062:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.356065:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.356067:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.356070:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.356075:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764909.356083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067912576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.356090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.356093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764909.356095:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.356098:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.356101:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.356115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.356119:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764909.356128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067912704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764909.356136:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764909.356138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764909.356140:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764909.356143:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2377 00000100:00080000:2.0:1789764909.356147:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764909.356150:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789764909.356153:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764909.356162:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067912832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764909.356297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067912064:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764909.356357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356391:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061dd500 x1876702067912064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067912064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067912064:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764909.356429:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764909.356452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356464:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061df480 x1876702067912192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356476:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067912192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356484:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764909.356495:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764909.356514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061df100 x1876702067912320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067912320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356543:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356546:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356549:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764909.356554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764909.356574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356585:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061dd880 x1876702067912448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356596:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067912448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764909.356618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764909.356636:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356647:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061ddc00 x1876702067912576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356657:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067912576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356664:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356667:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067912576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764909.356675:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067912704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764909.356694:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356705:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061de300 x1876702067912704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067912704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356741:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067912704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764909.356750:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067912832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764909.356770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764909.356782:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764909/real 1789764909] req@ffff9360061ddf80 x1876702067912832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764909.356794:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067912832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764934 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764909.356832:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764909.356836:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764909.356839:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067912832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764910.122082:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764911.146018:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764912.170180:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764913.195460:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764914.218110:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764914.475777:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.475786:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764914.475790:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.475988:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.476000:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.476007:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764914.476043:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067912960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764914.476117:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067912960:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764914.476175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764914.476205:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.476208:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764914.476210:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.476214:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.476218:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.476221:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.476230:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764914.476243:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067913088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764914.479237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020dac700 x1876702067912960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.479263:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067912960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.479283:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.479288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.479292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067912960:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764914.479323:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764914.479731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.479751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020daf100 x1876702067913088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.479764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067913088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.479772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.479776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.479779:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913088:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764914.480454:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.480460:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764914.480463:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.480468:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.480474:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.480481:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.480491:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764914.480518:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067913216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764914.480540:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.480543:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764914.480545:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.480549:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.480552:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.480555:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.480560:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764914.480569:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067913344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764914.480578:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.480581:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764914.480583:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.480586:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.480589:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.480593:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.480598:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764914.480606:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067913472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764914.480614:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.480617:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764914.480618:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.480622:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.480625:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.480628:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.480632:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764914.480640:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067913600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764914.480648:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764914.480651:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764914.480652:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764914.480656:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2382 00000100:00080000:0.0:1789764914.480659:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764914.480662:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764914.480666:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764914.480675:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067913728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764914.480882:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764914.480920:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.480937:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020dac380 x1876702067913216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.480949:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067913216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.480957:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.480961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.480965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764914.480971:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764914.480991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.481002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020dad180 x1876702067913344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.481012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067913344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.481020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.481023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.481026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764914.481032:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764914.481049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.481059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020dae300 x1876702067913472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.481080:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067913472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.481088:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.481091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.481094:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764914.481100:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067913600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764914.481119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.481129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020daf800 x1876702067913600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.481144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067913600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.481151:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.481154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.481157:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067913600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764914.481162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067913728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764914.481176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764914.481184:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764914/real 1789764914] req@ffff936020dadc00 x1876702067913728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764914.481194:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067913728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764914.481201:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764914.481204:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764914.481207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067913728:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764915.242275:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764916.266035:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764917.290085:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764918.316710:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764919.338995:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764919.595686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.595696:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764919.595699:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.595709:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.595719:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.595726:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764919.595762:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067913856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764919.595844:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067913856:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764919.595904:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764919.595948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.595951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764919.595953:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.595958:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.595962:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.595966:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.595974:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764919.595987:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067913984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764919.596070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa7100 x1876702067913856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.596087:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067913856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.596097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.596101:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.596104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067913856:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764919.596129:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764919.596151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764919.596178:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa5180 x1876702067913984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.596188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067913984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.596196:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.596198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.596201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067913984:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764919.596612:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.596617:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764919.596619:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.596623:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.596628:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.596632:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.596638:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764919.596653:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067914112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764919.596745:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764919.596769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764919.596790:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.596794:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764919.596796:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.596819:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.596823:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.596827:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.596833:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764919.596843:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067914240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764919.596943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa4380 x1876702067914112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.596957:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067914112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.596965:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.596969:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.596972:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764919.596988:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764919.597011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764919.597023:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa4a80 x1876702067914240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.597048:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067914240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.597057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.597060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.597063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914240:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764919.597076:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.597080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764919.597082:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.597086:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.597090:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.597153:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.597159:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764919.597172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067914368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764919.597191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.597194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764919.597196:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.597199:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.597204:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764919.597208:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.597212:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764919.597223:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067914496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764919.597249:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764919.597272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764919.597283:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa5880 x1876702067914368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.597294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067914368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.597301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.597304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.597307:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914368:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789764919.597376:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764919.597379:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764919.597381:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764919.597385:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2387 00000100:00080000:3.0:1789764919.597389:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789764919.597391:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067914496:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764919.597393:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764919.597398:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764919.597408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067914624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764919.597410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764919.597421:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa5c00 x1876702067914496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.597432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067914496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.597439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.597442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.597445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067914496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764919.597456:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067914624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764919.597473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764919.597480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764919/real 1789764919] req@ffff936010aa7800 x1876702067914624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764919.597487:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067914624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764919.597492:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764919.597495:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764919.597498:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067914624:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764920.361986:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764921.392267:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764922.410693:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764923.434972:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764924.458551:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764924.714164:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764924.714178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714188:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714199:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714205:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764924.714241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067914752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714280:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714284:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764924.714286:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714290:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714293:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714297:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714306:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764924.714318:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067914880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714327:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714330:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764924.714331:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714334:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714337:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714340:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714344:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764924.714353:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067915008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714361:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714364:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764924.714366:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714369:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714372:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714375:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714380:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764924.714389:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067915136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714396:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714399:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764924.714400:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714403:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714406:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714409:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714413:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764924.714420:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067915264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714427:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714430:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764924.714431:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714434:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714436:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714440:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714444:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764924.714452:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067915392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764924.714460:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764924.714463:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764924.714464:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764924.714467:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2392 00000100:00080000:1.0:1789764924.714470:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764924.714473:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764924.714477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764924.714487:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067915520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764924.714962:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067914752:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789764924.715028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715091:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef7100 x1876702067914752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067914752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715120:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715124:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067914752:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764924.715131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764924.715154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715166:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef4380 x1876702067914880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715176:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067914880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715183:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715185:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715188:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067914880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764924.715193:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764924.715212:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715222:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef4000 x1876702067915008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715233:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067915008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715241:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764924.715252:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764924.715272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715282:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef5c00 x1876702067915136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715293:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067915136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764924.715311:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789764924.715330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715340:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef5880 x1876702067915264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715360:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067915264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715367:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715370:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715373:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764924.715379:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067915392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764924.715399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef4a80 x1876702067915392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715420:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067915392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715434:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715437:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715440:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067915392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764924.715458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067915520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789764924.715479:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789764924.715495:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764924/real 1789764924] req@ffff936010ef5180 x1876702067915520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764924.715507:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067915520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764924.715515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764924.715518:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764924.715521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067915520:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764925.482519:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764926.506016:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764927.531487:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764928.555199:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764929.580506:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764929.834409:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834487:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764929.834491:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834501:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834511:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834519:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764929.834555:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067915648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834588:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834592:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764929.834593:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834597:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834601:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834605:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834613:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764929.834636:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067915776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834647:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834649:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764929.834651:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834654:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834657:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834660:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834664:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764929.834673:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067915904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834682:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834685:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764929.834687:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834690:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834693:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834696:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834700:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764929.834707:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067916032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834721:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834723:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764929.834725:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834728:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834730:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834733:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834737:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764929.834744:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067916160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834751:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834754:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764929.834756:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834759:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834761:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834765:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834768:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764929.834775:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067916288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764929.834783:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764929.834786:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764929.834788:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764929.834791:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2397 00000100:00080000:1.0:1789764929.834794:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764929.834813:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764929.834818:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764929.834826:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067916416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764929.835157:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067915648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764929.836047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764929.836081:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764929.836103:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764929.836124:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764929.836141:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067916288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764929.836160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067916416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764930.601989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764931.628756:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764932.650205:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789764932.908307:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789764932.908363:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789764932.908380:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908419:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908428:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908436:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908444:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908451:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789764932.908458:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764932.908710:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef4700 x1876702067915648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.908766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067915648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.908780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.908794:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.908824:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067915648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764932.908832:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef5180 x1876702067915776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.908847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067915776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.908855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.908859:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.908862:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764932.908867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef4a80 x1876702067915904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.908874:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067915904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.908880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.908965:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.908970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067915904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764932.908977:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef5880 x1876702067916032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.908991:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067916032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.908999:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.909002:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.909005:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764932.909012:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef5c00 x1876702067916160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.909022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067916160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.909030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.909084:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.909091:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764932.909098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef4000 x1876702067916288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.909112:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067916288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.909120:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.909123:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.909126:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067916288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764932.909132:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764929/real 1789764932] req@ffff936010ef4380 x1876702067916416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764932.909142:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067916416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764932.909151:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764932.909154:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764932.909157:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067916416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764933.675848:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764934.699005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764934.955862:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.955873:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764934.955877:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.955888:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.955899:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.955906:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764934.956075:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067916544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764934.956139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067916544:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764934.956218:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789764934.956249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.956254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764934.956256:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.956260:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.956264:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.956268:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.956278:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764934.956292:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067916672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764934.957989:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa7b80 x1876702067916544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.958018:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067916544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.958028:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.958031:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.958034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067916544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764934.958058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764934.958099:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.958110:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa6300 x1876702067916672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.958123:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067916672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.958132:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.958134:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.958136:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916672:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789764934.959117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.959124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764934.959127:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.959132:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.959138:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.959143:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.959153:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764934.959176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067916800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764934.959186:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.959188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764934.959189:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.959192:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.959194:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.959196:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.959198:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764934.959205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067916928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764934.959298:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.959301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764934.959302:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.959305:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.959308:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.959310:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.959313:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764934.959540:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067917056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764934.959559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.959562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764934.959563:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.959565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.959568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.959570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.959573:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764934.959590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067917184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764934.959613:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764934.959615:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764934.959616:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764934.959618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2402 00000100:00080000:3.0:1789764934.959620:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764934.959622:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764934.959625:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764934.959641:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067917312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764934.959769:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764934.959870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.959906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa4000 x1876702067916800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.959931:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067916800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.959941:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.959945:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.959949:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764934.959956:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764934.959976:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.960179:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa6300 x1876702067916928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.960188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067916928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.960249:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.960252:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.960256:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067916928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764934.960263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764934.960285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.960297:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa7b80 x1876702067917056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.960308:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067917056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.960315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.960318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.960321:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764934.960327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067917184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764934.962131:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.962161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa7800 x1876702067917184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.962179:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067917184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.962189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.962386:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.962391:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067917184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764934.962403:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067917312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764934.962431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764934.962445:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764934/real 1789764934] req@ffff936010aa5c00 x1876702067917312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764934.962458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067917312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764934.962467:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764934.962471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764934.962474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067917312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764935.722288:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764936.746139:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764937.770359:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764938.794879:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764939.819989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764940.073961:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764940.073969:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764940.073973:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764940.073982:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.073991:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.073997:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764940.074029:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067917440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764940.074056:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764940.074059:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764940.074061:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764940.074064:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074067:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.074071:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074080:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764940.074126:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067917440:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764940.074182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764940.074222:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020dace00 x1876702067917440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764940.074238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067917440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764940.074248:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.074252:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.074255:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067917440:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789764940.074602:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067917568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764940.074624:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764940.074627:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764940.074629:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764940.074633:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074637:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.074640:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074645:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764940.074655:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067917696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764940.074666:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789764940.074669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917568:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764940.074670:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764940.074674:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764940.074679:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074684:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.074687:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074692:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789764940.074706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764940.074707:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067917824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764940.074717:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764940.074720:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764940.074723:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789764940.074724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020dadf80 x1876702067917568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764940.074731:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074738:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.074743:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074747:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764940.074747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067917568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764940.074757:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067917952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764940.074760:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.074769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764940.074774:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789764940.074775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917568:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764940.074778:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764940.074790:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789764940.074792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917696:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764940.074822:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074826:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764940.074831:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074836:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764940.074850:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067918080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764940.074860:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789764940.074860:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764940.074866:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764940.074868:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764940.074872:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2407 00000100:00080000:0.0:1789764940.074876:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789764940.074877:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020daea00 x1876702067917696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764940.074892:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764940.074900:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764940.074907:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067917696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764940.074914:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067918208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764940.075138:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.075143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.075147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764940.075154:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764940.075178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764940.075189:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020dace00 x1876702067917824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764940.075200:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067917824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764940.075207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.075210:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.075213:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764940.075219:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764940.075235:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764940.075245:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020dadc00 x1876702067917952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764940.075254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067917952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764940.075260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.075263:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.075266:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067917952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764940.075279:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764940.075298:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764940.075307:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020daf800 x1876702067918080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764940.075317:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067918080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764940.075324:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.075326:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.075329:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764940.075334:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764940.075351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764940.075361:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764940/real 1789764940] req@ffff936020dae300 x1876702067918208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764940.075372:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067918208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764940.075378:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764940.075381:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764940.075384:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764940.841999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764941.866013:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764942.890984:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764943.915928:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764944.938082:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764945.194093:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764945.194106:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194116:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194125:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194131:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764945.194166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067918336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764945.194191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194194:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764945.194196:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194199:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194207:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194216:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764945.194229:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067918464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764945.194237:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067918336:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764945.194241:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194246:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764945.194248:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194253:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194256:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194260:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194265:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764945.194276:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067918592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764945.194285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764945.194289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194292:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194295:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194298:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194302:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789764945.194307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789764945.194313:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067918720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764945.194324:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194327:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764945.194329:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194332:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194336:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194340:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194344:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764945.194353:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067918848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764945.194360:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dad180 x1876702067918336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764945.194364:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789764945.194384:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067918336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.194395:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.194399:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.194402:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067918336:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764945.194423:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764945.194445:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.194458:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dac380 x1876702067918464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.194476:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067918464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.194484:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.194487:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.194490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918464:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764945.194656:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764945.194658:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194663:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194668:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194672:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194677:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764945.194689:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067918976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764945.194700:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764945.194703:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764945.194705:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764945.194708:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2413 00000100:00080000:0.0:1789764945.194712:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764945.194716:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764945.194721:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764945.194732:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067919104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764945.195740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764945.195781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.195827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020daf100 x1876702067918592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.195853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067918592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.195862:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.195866:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.195869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764945.195876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764945.195896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.195906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dac700 x1876702067918720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.195917:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067918720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.195929:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.195932:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.195940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764945.195946:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764945.195966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.195977:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dafb80 x1876702067918848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.195987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067918848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.195994:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.195997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.196000:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067918848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764945.196022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764945.196043:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.196054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dac000 x1876702067918976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.196065:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067918976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.196072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.196075:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.196078:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067918976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764945.196084:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067919104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764945.196115:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764945.196127:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764945/real 1789764945] req@ffff936020dad880 x1876702067919104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764945.196137:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067919104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764945.196143:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764945.196146:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764945.196149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067919104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764945.962086:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764946.986384:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764948.009983:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764949.034033:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764950.058095:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764950.314058:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.314067:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764950.314072:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.314081:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00080000:0.0:1789764950.314093:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.314100:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764950.314136:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067919232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764950.314217:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067919232:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764950.314274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764950.314305:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.314309:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764950.314310:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.314314:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00080000:0.0:1789764950.314319:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.314323:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764950.314332:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764950.314345:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067919360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764950.315101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020dad500 x1876702067919232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764950.315124:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067919232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764950.315135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067919232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764950.315165:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919360:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764950.315176:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.315188:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764950.315190:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.315195:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00080000:0.0:1789764950.315201:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.315205:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789764950.315205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789764950.315211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764950.315218:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020daf480 x1876702067919360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764950.315225:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067919488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764950.315229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067919360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764950.315236:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764950.315237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764950.315239:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764950.315241:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764950.315241:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789764950.315244:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00100000:1.0:1789764950.315244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919360:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764950.315247:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.315251:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764950.315255:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764950.315255:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789764950.315263:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067919616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764950.315272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764950.315273:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.315275:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764950.315277:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.315280:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00100000:1.0:1789764950.315282:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020dae680 x1876702067919488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764950.315283:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.315287:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764950.315291:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764950.315292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067919488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764950.315299:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067919744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764950.315300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919488:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764950.315308:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.315311:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764950.315312:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.315316:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00100000:1.0:1789764950.315316:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919616:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764950.315319:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.315322:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764950.315326:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789764950.315333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764950.315343:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020daed80 x1876702067919616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764950.315360:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067919616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764950.315368:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315370:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315373:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764950.315390:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764950.315409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764950.315420:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020daf480 x1876702067919744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764950.315430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067919744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764950.315438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315444:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067919744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789764950.315523:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067919872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764950.315542:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764950.315545:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764950.315547:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764950.315550:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2418 00000100:00080000:0.0:1789764950.315554:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764950.315558:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764950.315563:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764950.315572:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067920000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764950.315640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067919872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764950.315667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764950.315679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020dad500 x1876702067919872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764950.315690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067919872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764950.315704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315707:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067919872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764950.315716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764950.315734:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764950.315745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764950/real 1789764950] req@ffff936020dad880 x1876702067920000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764950.315754:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067920000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764975 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764950.315761:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764950.315764:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764950.315767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764951.082051:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764952.109249:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764953.131812:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764954.154023:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764955.177932:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764955.436397:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436406:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764955.436410:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436420:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436431:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436438:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764955.436474:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067920128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436508:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436512:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764955.436514:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436518:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436522:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436526:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436535:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764955.436548:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067920256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436558:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436562:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764955.436564:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436568:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436572:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436575:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436580:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764955.436591:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067920384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436601:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436604:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764955.436606:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436609:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436612:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436616:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436620:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764955.436629:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067920512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436638:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436641:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764955.436643:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436646:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436650:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436653:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436657:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764955.436665:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067920640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436673:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436677:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764955.436678:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436681:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436684:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436688:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436692:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764955.436700:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067920768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764955.436707:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764955.436710:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764955.436712:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764955.436715:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2423 00000100:00080000:3.0:1789764955.436718:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764955.436721:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764955.436725:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764955.436733:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067920896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764955.436841:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067920128:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764955.436898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.436936:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa5880 x1876702067920128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.436952:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067920128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.436962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.436966:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.436970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067920128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764955.436976:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764955.436999:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437012:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa4a80 x1876702067920256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437025:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067920256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764955.437046:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764955.437064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437076:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa4380 x1876702067920384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067920384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437095:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764955.437108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764955.437127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa5180 x1876702067920512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067920512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764955.437172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764955.437191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa7100 x1876702067920640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067920640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437220:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437225:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067920640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764955.437231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764955.437248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437259:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa7480 x1876702067920768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437270:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067920768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437285:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764955.437291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764955.437308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764955.437317:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764955/real 1789764955] req@ffff936010aa6680 x1876702067920896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764955.437328:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067920896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764955.437336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764955.437339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764955.437341:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067920896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789764956.203627:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764957.226330:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764958.250590:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764959.275188:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764960.299093:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764960.553984:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.553993:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764960.553997:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554007:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554018:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554025:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764960.554061:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067921024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554094:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554098:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764960.554100:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554103:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554107:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554111:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554119:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764960.554130:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067921152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554142:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764960.554143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554147:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554150:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554152:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764960.554164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067921280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764960.554175:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554178:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554181:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554183:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554187:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764960.554194:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067921408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554201:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554203:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764960.554204:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554207:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554210:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554212:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554215:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764960.554222:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067921536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554233:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764960.554235:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554238:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554241:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554245:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554249:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764960.554259:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067921664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764960.554267:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764960.554269:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764960.554271:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764960.554274:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2428 00000100:00080000:3.0:1789764960.554277:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764960.554280:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764960.554283:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764960.554292:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067921792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764960.554356:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067921024:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764960.554410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa5500 x1876702067921024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067921024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554474:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067921024:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764960.554485:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764960.554506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554518:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa6a00 x1876702067921152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067921152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554537:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764960.554549:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764960.554567:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa4700 x1876702067921280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067921280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764960.554610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764960.554630:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554640:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa4e00 x1876702067921408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067921408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554660:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764960.554671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764960.554689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa5f80 x1876702067921536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554710:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067921536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067921536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764960.554746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067921664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764960.554765:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554776:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa6680 x1876702067921664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554793:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067921664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554830:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554837:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067921664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764960.554843:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067921792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764960.554863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764960.554873:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764960/real 1789764960] req@ffff936010aa7480 x1876702067921792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764960.554884:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067921792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764960.554891:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764960.554894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764960.554897:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067921792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764961.321930:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764962.347376:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764963.369990:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764964.393961:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764965.418331:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764965.677072:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677084:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764965.677089:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677101:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677114:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677122:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789764965.677168:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067921920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677210:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677214:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764965.677215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677219:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677223:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677227:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677246:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764965.677259:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067922048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677268:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677272:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764965.677273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677277:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677284:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677288:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677293:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764965.677303:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067922176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677313:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677316:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764965.677317:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677320:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677324:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677327:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677331:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764965.677340:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067922304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677347:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677350:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789764965.677352:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677355:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677359:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677362:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677366:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764965.677374:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067922432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677381:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677384:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764965.677386:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677388:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677391:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677395:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677399:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764965.677412:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067922560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764965.677426:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764965.677429:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764965.677430:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764965.677434:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2433 00000100:00080000:3.0:1789764965.677438:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764965.677441:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789764965.677446:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764965.677460:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067922688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764965.677669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067921920:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764965.677769:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678008:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa7100 x1876702067921920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.678029:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067921920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.678041:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.678045:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.678049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067921920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764965.678057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764965.678083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678097:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa5180 x1876702067922048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.678110:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067922048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.678119:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.678122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.678125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764965.678132:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764965.678152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa4380 x1876702067922176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.678175:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067922176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.678184:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.678187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.678190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764965.678196:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764965.678219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa4a80 x1876702067922304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.678388:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067922304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.678397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.678400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.678403:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764965.678409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764965.678600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa5880 x1876702067922432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.678626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067922432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.678634:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.678637:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.678640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764965.678647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067922560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764965.678667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.678679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa5c00 x1876702067922560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.683527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067922560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.683560:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.683571:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.683579:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067922560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764965.683595:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067922688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764965.683701:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764965.683741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764965/real 1789764965] req@ffff936010aa7800 x1876702067922688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764965.683758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067922688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764965.683775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764965.683904:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764965.683909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067922688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789764966.446971:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764967.466060:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789764968.495408:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764969.514211:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764970.537965:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764970.794569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764970.794582:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794591:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794601:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794608:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764970.794646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067922816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764970.794693:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794697:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794701:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794704:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794713:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764970.794725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067922944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764970.794752:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764970.794777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067923072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764970.794790:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794794:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794827:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794831:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794836:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764970.794845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067923200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764970.794858:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794861:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794864:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794868:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764970.794881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067923328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794892:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764970.794894:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794897:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794900:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764970.794917:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067923456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764970.794926:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764970.794929:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764970.794931:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764970.794934:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2438 00000100:00080000:0.0:1789764970.794937:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764970.794940:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764970.794944:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764970.794953:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067923584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764970.795086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067922816:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764970.795140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795190:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dac000 x1876702067922816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067922816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795215:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067922816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764970.795229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764970.795252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795264:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dafb80 x1876702067922944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795275:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067922944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795282:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795285:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795288:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067922944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764970.795294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764970.795310:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dac700 x1876702067923072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795329:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067923072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764970.795347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764970.795366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795377:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020daf100 x1876702067923200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795388:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067923200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795396:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795399:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795402:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764970.795407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764970.795424:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dac380 x1876702067923328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795444:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067923328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795451:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795457:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764970.795462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067923456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764970.795477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dad180 x1876702067923456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067923456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795504:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795507:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067923456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764970.795514:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067923584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764970.795529:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764970.795537:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764970/real 1789764970] req@ffff936020dae300 x1876702067923584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764970.795548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067923584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764970.795555:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764970.795558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764970.795561:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067923584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764971.561996:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764972.587373:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764973.612263:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764974.634684:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764975.657991:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764975.913883:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.913890:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764975.913893:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.913901:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.913911:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.913916:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764975.913944:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067923712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764975.914008:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067923712:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764975.914055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764975.914079:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914082:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764975.914083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914086:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914089:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914092:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914099:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764975.914108:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067923840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764975.914156:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020daf800 x1876702067923712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.914168:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067923712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.914178:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.914181:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.914183:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067923712:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764975.914206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764975.914222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764975.914230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020dadc00 x1876702067923840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.914237:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067923840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.914242:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.914244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.914246:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923840:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789764975.914439:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914445:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764975.914531:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914535:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914539:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914542:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914546:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764975.914555:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067923968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764975.914587:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764975.914605:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764975.914636:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914638:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764975.914640:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914643:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914646:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914649:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914653:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764975.914661:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067924096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764975.914692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020dace00 x1876702067923968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.914707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067923968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.914712:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.914714:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.914716:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067923968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764975.914725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764975.914740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764975.914747:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020daea00 x1876702067924096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.914753:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067924096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.914758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.914761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.914762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924096:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789764975.914793:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914796:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764975.914812:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914815:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914818:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914820:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914824:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764975.914832:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067924224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764975.914838:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914840:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764975.914842:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914843:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914845:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914847:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914849:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764975.914855:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067924352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764975.914859:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764975.914861:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764975.914862:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764975.914864:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2443 00000100:00080000:0.0:1789764975.914866:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764975.914868:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764975.914870:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764975.914875:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067924480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764975.915000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764975.915016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764975.915024:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020dadf80 x1876702067924224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.915031:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067924224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.915036:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.915038:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.915040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764975.915043:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067924352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764975.915053:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764975.915059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020daea00 x1876702067924352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.915066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067924352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.915070:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.915071:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.915073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067924352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764975.915076:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067924480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764975.915086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764975.915091:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764975/real 1789764975] req@ffff936020dace00 x1876702067924480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764975.915097:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067924480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764975.915101:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764975.915103:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764975.915104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067924480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764976.682014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764977.706084:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764978.730060:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764979.756651:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764980.778510:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764981.034280:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764981.034292:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034302:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034312:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034318:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764981.034352:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067924608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034379:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034383:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764981.034385:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034388:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034392:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034395:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.034404:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764981.034415:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067924736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034425:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034428:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764981.034430:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034434:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034438:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034441:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.034445:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764981.034456:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067924864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034465:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034468:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764981.034470:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034473:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034477:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034480:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.034484:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764981.034495:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067924992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034503:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034506:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764981.034508:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034511:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034514:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034517:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.034522:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764981.034532:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067925120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034540:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034543:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764981.034544:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.034547:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.034550:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.034554:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.034557:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764981.034566:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067925248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764981.034575:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764981.034578:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764981.036499:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764981.036508:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2448 00000100:00080000:0.0:1789764981.036518:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764981.036524:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764981.036534:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764981.036559:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067925376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764981.036833:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067924608:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764981.036900:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.036938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020dadc00 x1876702067924608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.036954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067924608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.036970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.036973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037092:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067924608:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764981.037101:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764981.037126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020daf800 x1876702067924736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037150:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067924736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037159:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764981.037167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764981.037186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037196:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020dae300 x1876702067924864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037207:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067924864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037215:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037221:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764981.037227:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764981.037259:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037270:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020dad180 x1876702067924992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037282:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067924992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037290:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037294:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037296:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067924992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764981.037303:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764981.037322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020dac380 x1876702067925120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067925120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764981.037363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067925248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764981.037382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020daf100 x1876702067925248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067925248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037412:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037415:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037418:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067925248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764981.037422:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067925376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764981.037440:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764981.037450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764981/real 1789764981] req@ffff936020dac700 x1876702067925376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764981.037461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067925376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764981.037468:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764981.037470:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764981.037473:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067925376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764981.802031:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764982.826481:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764983.850713:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764984.877855:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764985.898026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764986.154396:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154407:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764986.154412:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154421:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154432:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154439:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764986.154491:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067925504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764986.154524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154528:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764986.154531:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154535:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154539:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154551:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154560:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764986.154572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067925632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764986.154582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764986.154586:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154589:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00100000:1.0:1789764986.154591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067925504:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789764986.154593:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154596:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154600:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764986.154608:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067925760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764986.154622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764986.154626:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154702:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154706:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154711:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764986.154729:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067925888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764986.154737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764986.154742:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154746:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154749:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154752:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154756:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764986.154765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067926016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789764986.154766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789764986.154773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154775:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764986.154777:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154780:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154791:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764986.154815:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067926144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764986.154828:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764986.154831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764986.154832:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764986.154836:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2453 00000100:00080000:0.0:1789764986.154839:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764986.154842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764986.154847:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764986.154856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067926272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764986.154878:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020dafb80 x1876702067925504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.154895:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067925504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.154911:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.154915:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.154919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067925504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764986.154925:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764986.154950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.154963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020dac000 x1876702067925632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.154974:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067925632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.154980:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.154983:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.154986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764986.155007:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764986.155026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.155037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020dad880 x1876702067925760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.155046:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067925760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.155053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.155056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.155058:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764986.155065:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764986.155084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.155095:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020dad500 x1876702067925888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.155105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067925888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.155111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.155114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.155117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067925888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764986.155121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764986.155153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.155164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020daf480 x1876702067926016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.155175:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067926016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.155182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.155185:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.155188:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764986.155719:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067926144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764986.155752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.155767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020daed80 x1876702067926144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.155779:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067926144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.155787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.155790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.155793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067926144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764986.155824:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067926272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764986.155846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764986.155857:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764986/real 1789764986] req@ffff936020dae680 x1876702067926272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764986.155868:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067926272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764986.155876:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764986.155879:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764986.155882:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067926272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764986.922703:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764987.950200:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764988.970009:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764989.994620:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764991.018561:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764991.275559:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275568:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764991.275571:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275579:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275589:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275595:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789764991.275630:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067926400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275659:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275663:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764991.275665:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275669:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275672:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275676:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275685:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764991.275696:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067926528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275705:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275708:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764991.275710:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275714:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275717:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275721:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275725:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764991.275735:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067926656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275743:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275746:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764991.275747:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275750:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275753:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275756:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275761:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764991.275769:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067926784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275777:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275780:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789764991.275782:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275785:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275787:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275791:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275795:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764991.275823:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067926912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275831:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275834:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764991.275835:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275838:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275841:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275845:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275850:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764991.275858:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067927040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764991.275866:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764991.275868:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764991.275870:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764991.275874:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2459 00000100:00080000:0.0:1789764991.275876:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764991.275880:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789764991.275884:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764991.275893:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067927168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789764991.276075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067926400:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789764991.276204:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276245:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020dae680 x1876702067926400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276264:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067926400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276279:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276283:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276287:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067926400:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789764991.276295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764991.276320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276333:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020daed80 x1876702067926528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276344:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067926528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276354:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276358:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764991.276363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764991.276382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276393:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020daf480 x1876702067926656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276405:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067926656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276413:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276416:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276418:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764991.276424:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764991.276442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020dad500 x1876702067926784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276463:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067926784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764991.276480:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789764991.276496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276516:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020dad880 x1876702067926912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067926912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276534:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276537:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067926912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789764991.276546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764991.276564:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276575:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020dac000 x1876702067927040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276585:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067927040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276592:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276597:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789764991.276602:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789764991.276620:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789764991.276631:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764991/real 1789764991] req@ffff936020dafb80 x1876702067927168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789764991.276642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067927168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765016 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789764991.276648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789764991.276651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789764991.276654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764992.042051:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764993.066000:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764994.089976:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764995.115243:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764996.138194:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789764996.393940:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.393950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789764996.393954:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.393964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.393974:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.393982:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789764996.394017:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067927296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764996.394066:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394070:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394074:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394079:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394088:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764996.394110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067927424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764996.394125:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394129:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394132:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394136:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394140:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764996.394150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067927552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789764996.394164:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394168:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394172:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394175:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394180:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789764996.394190:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067927680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789764996.394204:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394207:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394210:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394214:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394218:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764996.394240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067927808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789764996.394254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394260:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394263:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764996.394277:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067927936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789764996.394291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789764996.394294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789764996.394296:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789764996.394299:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2464 00000100:00080000:1.0:1789764996.394302:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789764996.394305:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789764996.394309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789764996.394319:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067928064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764996.394418:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067927296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764996.394559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764996.394592:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764996.394617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764996.394643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764996.394662:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764996.394680:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789764997.162087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764998.186115:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764999.210344:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789764999.466498:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:0.0:1789764999.466560:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789764999.466580:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466620:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466629:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466638:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466644:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466650:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789764999.466656:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789764999.466728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef7100 x1876702067927296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067927296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466780:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466784:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067927296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789764999.466793:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef4e00 x1876702067927424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466841:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067927424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466849:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466853:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466857:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764999.466873:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef6d80 x1876702067927552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067927552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466895:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764999.466908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef6300 x1876702067927680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466918:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067927680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466924:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466927:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466931:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764999.466938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef7b80 x1876702067927808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466949:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067927808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466959:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466962:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067927808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789764999.466969:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef7480 x1876702067927936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.466980:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067927936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.466988:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.466991:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.466994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067927936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789764999.467001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764996/real 1789764999] req@ffff936010ef6680 x1876702067928064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764999.467012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067928064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765021 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764999.467019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764999.467022:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789764999.467025:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765000.234175:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765001.257990:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765001.520593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765001.520610:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520619:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520632:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520650:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765001.520688:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067928192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765001.520731:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520742:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765001.520762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067928320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765001.520776:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520780:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520783:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765001.520844:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067928448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520854:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520857:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765001.520859:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520862:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520865:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520869:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520874:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765001.520883:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067928576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520894:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765001.520896:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520899:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520901:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520904:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765001.520916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067928704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765001.520930:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520933:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520936:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520939:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520944:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765001.520952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067928832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765001.520959:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765001.520962:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765001.520964:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765001.520967:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2469 00000100:00080000:0.0:1789765001.520970:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765001.520973:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765001.520977:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765001.520986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067928960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765001.521288:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067928192:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765001.521351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020dac700 x1876702067928192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067928192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521419:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521422:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067928192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765001.521443:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765001.521465:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020daf100 x1876702067928320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521489:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067928320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521497:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765001.521509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765001.521527:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521537:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020dac380 x1876702067928448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067928448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521555:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521561:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765001.521566:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765001.521584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020dad180 x1876702067928576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521605:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067928576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521618:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521623:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765001.521629:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765001.521647:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521657:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020dae300 x1876702067928704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067928704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521675:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521678:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521680:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067928704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765001.521686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765001.521705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521716:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020daf800 x1876702067928832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067928832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765001.521761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765001.521781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765001.521792:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765001/real 1789765001] req@ffff936020dadc00 x1876702067928960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765001.521835:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067928960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765001.521843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765001.521846:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765001.521849:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067928960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765002.282017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765003.306642:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765004.330131:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765005.354008:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765006.378818:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765006.633914:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.633924:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765006.633928:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.633937:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00080000:0.0:1789765006.633946:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.633954:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765006.633989:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067929088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765006.634021:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.634024:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765006.634026:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634030:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00080000:0.0:1789765006.634033:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634037:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634045:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765006.634058:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067929216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765006.634075:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067929088:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765006.634077:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.634084:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765006.634086:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634091:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00080000:0.0:1789765006.634095:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634098:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634102:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765006.634114:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067929344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765006.634123:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.634126:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765006.634128:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634132:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00080000:0.0:1789765006.634145:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634149:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789765006.634155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634157:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765006.634176:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067929472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765006.634186:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.634189:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765006.634191:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634197:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00100000:1.0:1789765006.634197:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020dace00 x1876702067929088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765006.634204:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634211:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765006.634224:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067929088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765006.634232:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067929600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765006.634239:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765006.634247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765006.634248:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765006.634255:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789765006.634256:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067929088:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765006.634258:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634265:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2474 00000100:00080000:0.0:1789765006.634269:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634274:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634279:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765006.634285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765006.634291:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067929728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765006.634303:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765006.634307:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765006.634309:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765006.634315:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2474 00000800:00000100:1.0:1789765006.634316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765006.634321:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765006.634326:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765006.634332:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765006.634336:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020daea00 x1876702067929216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765006.634346:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067929856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765006.634355:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067929216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634385:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634408:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765006.634416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765006.634438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765006.634450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020dadf80 x1876702067929344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765006.634461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067929344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634484:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634488:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765006.634494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765006.634512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765006.634522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020dadc00 x1876702067929472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765006.634532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067929472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765006.634551:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765006.634570:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765006.634581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020daf800 x1876702067929600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765006.634592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067929600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067929600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765006.634620:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067929728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765006.634639:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765006.634650:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020dae300 x1876702067929728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765006.634660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067929728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634671:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634674:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067929728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765006.634680:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067929856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765006.634696:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765006.634706:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765006/real 1789765006] req@ffff936020dad180 x1876702067929856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765006.634716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067929856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765006.634724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765006.634726:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765006.634729:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067929856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765007.402002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765008.426047:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765009.449946:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765010.477878:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765011.498083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765011.754025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765011.754043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765011.754047:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754056:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754066:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754072:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765011.754106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067929984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765011.754138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765011.754140:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754144:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754147:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754160:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765011.754172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067930112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765011.754184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765011.754185:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754191:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00100000:1.0:1789765011.754192:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067929984:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765011.754195:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754202:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754206:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765011.754217:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067930240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765011.754226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765011.754227:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754230:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754233:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754235:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754238:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765011.754245:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067930368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789765011.754253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765011.754254:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765011.754258:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754261:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754264:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754266:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754269:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765011.754275:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067930496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765011.754282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765011.754283:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754285:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754287:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754291:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754295:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765011.754300:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061dc700 x1876702067929984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765011.754308:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067930624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765011.754321:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765011.754322:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067929984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765011.754327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765011.754331:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765011.754339:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2479 00000100:00080000:2.0:1789765011.754342:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765011.754344:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765011.754348:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789765011.754350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067929984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765011.754371:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067930752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765011.754380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765011.754403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061dc380 x1876702067930112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067930112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754546:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754549:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765011.754570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765011.754588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754598:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061ddf80 x1876702067930240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754609:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067930240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754617:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754623:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765011.754628:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765011.754642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754650:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061de300 x1876702067930368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067930368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754671:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754673:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765011.754678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765011.754691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061ddc00 x1876702067930496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067930496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754722:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067930496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765011.754726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067930624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765011.754740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061dd880 x1876702067930624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067930624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067930624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765011.754776:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067930752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765011.754789:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765011.754818:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765011/real 1789765011] req@ffff9360061df100 x1876702067930752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765011.754829:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067930752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765011.754837:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765011.754840:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765011.754843:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067930752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765012.523307:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765013.546289:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765014.570005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765015.596203:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765016.619550:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765016.874078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765016.874090:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874100:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874109:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874116:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765016.874153:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067930880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765016.874188:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874192:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874196:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874200:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874209:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765016.874220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067931008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765016.874233:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874237:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874240:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874244:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874248:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765016.874258:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067931136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765016.874272:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874275:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874278:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874282:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874286:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765016.874296:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067931264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874306:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874309:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765016.874311:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874315:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874328:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789765016.874331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067930880:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765016.874334:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874343:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765016.874355:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067931392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874369:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765016.874371:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874374:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874378:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874381:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874385:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789765016.874399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765016.874408:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067931520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765016.874419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765016.874422:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765016.874424:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765016.874428:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2484 00000100:00080000:1.0:1789765016.874432:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765016.874436:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765016.874442:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765016.874447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef7800 x1876702067930880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765016.874457:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067931648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765016.874471:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067930880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.874483:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.874488:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.874492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067930880:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765016.874503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765016.874528:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.874541:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef5500 x1876702067931008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.874554:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067931008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.874562:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.874565:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.874568:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765016.874575:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765016.874598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.874611:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef6a00 x1876702067931136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.874622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067931136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.874628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.874631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.874634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765016.874639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765016.875251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.875280:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef5f80 x1876702067931264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.875293:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067931264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.875302:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.875305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.875308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765016.875350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765016.875377:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.875391:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef6680 x1876702067931392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.875406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067931392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.875414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.875418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.875422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765016.875429:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067931520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765016.875453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.875464:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef7480 x1876702067931520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.875482:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067931520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.875496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.875500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.875504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067931520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765016.875509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067931648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765016.875528:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765016.875538:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765016/real 1789765016] req@ffff936010ef7b80 x1876702067931648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765016.875549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067931648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765016.875556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765016.875560:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765016.875563:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067931648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765017.642005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765018.666417:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765019.693998:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765020.714040:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765021.738206:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765021.995008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765021.995022:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995031:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995041:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995049:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765021.995085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067931776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765021.995164:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067931776:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765021.995224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765021.995327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765021.995333:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995338:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995343:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995347:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.995356:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765021.995370:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067931904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765021.995619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061df480 x1876702067931776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.995638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067931776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.995649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.995653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.995657:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067931776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765021.995682:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765021.995708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.995721:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061dd500 x1876702067931904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.995733:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067931904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.995741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.995744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.995746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067931904:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765021.995827:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765021.995833:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995837:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995841:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995846:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.995851:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765021.995861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067932032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765021.995868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765021.995871:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995873:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995875:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995877:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.995880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765021.995898:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067932160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765021.995914:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765021.995919:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995923:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995926:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995929:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.995933:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765021.995943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067932288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765021.995951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765021.995956:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995959:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995962:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.995966:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.995970:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765021.995978:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067932416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765021.995986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765021.995990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765021.995991:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765021.995994:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2489 00000100:00080000:2.0:1789765021.995997:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765021.996000:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765021.996004:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765021.996013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067932544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765021.996137:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765021.996162:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.996175:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061dc000 x1876702067932032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.996186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067932032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.996194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.996198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.996201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765021.996206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765021.996226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.996237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061dce00 x1876702067932160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.996334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067932160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.996342:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.996345:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.996348:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765021.996353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765021.996376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.996554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061dd180 x1876702067932288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.996567:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067932288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.996574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.996577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.996580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765021.996587:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067932416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765021.996609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.996621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061dca80 x1876702067932416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.996632:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067932416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.996639:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.996642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.996645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067932416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765021.996650:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067932544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765021.996669:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765021.996679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765021/real 1789765021] req@ffff9360061df800 x1876702067932544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765021.996689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067932544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765021.996697:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765021.996700:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765021.996702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067932544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765022.764866:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765023.786413:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765024.811981:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765025.835159:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765026.860220:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765027.117215:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.117225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765027.117229:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.117239:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.117259:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.117266:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765027.117306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067932672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765027.118320:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067932672:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765027.118380:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765027.119430:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.119436:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765027.119439:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.119446:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.119454:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.119459:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.119472:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765027.119496:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067932800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765027.119604:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dac380 x1876702067932672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765027.119624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067932672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765027.119634:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765027.119638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765027.119641:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067932672:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765027.119664:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765027.119699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765027.119715:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020daf100 x1876702067932800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765027.119727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067932800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765027.119735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765027.119738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765027.119741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932800:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765027.123670:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.123680:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765027.123683:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.123692:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.123701:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.123708:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.123720:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765027.126318:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067932928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765027.126703:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765027.127153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765027.127186:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dac700 x1876702067932928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765027.127204:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067932928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765027.127215:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765027.127219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765027.127223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067932928:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765027.127655:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.127660:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765027.127662:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.127667:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.127673:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.127678:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.127687:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765027.128732:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067933056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765027.128840:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.128844:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765027.128846:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.128850:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.128855:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.128859:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.128865:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765027.128878:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067933184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765027.128887:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.128890:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765027.128893:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.128896:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.128899:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.128902:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.128906:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765027.128913:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765027.128915:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067933312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765027.128927:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765027.128930:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765027.128931:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765027.128934:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2494 00000100:00080000:0.0:1789765027.128937:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765027.128941:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765027.128945:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765027.128953:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067933440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789765027.128963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765027.128984:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dafb80 x1876702067933056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765027.128998:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067933056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765027.129007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765027.129011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765027.129014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765027.129021:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765027.129042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765027.129053:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dac000 x1876702067933184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765027.129062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067933184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765027.129071:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765027.129074:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765027.129077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765027.129093:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067933312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765027.129109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765027.129118:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dad880 x1876702067933312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765027.129127:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067933312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765027.129135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765027.129138:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765027.129140:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067933312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765027.129144:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067933440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765027.129158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765027.129167:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765027/real 1789765027] req@ffff936020dad500 x1876702067933440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765027.129183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067933440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765027.129189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765027.129192:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765027.129195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067933440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765027.883267:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765028.906033:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765029.932605:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765030.954155:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765031.978101:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765032.233983:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.233992:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765032.233996:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234005:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234014:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234021:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765032.234058:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067933568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234089:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234093:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765032.234095:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234099:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234103:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234106:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234116:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765032.234129:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067933696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234138:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234141:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765032.234142:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234145:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234148:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234152:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765032.234166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067933824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234175:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234177:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765032.234179:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234182:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234187:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234190:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765032.234202:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067933952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234219:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234222:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765032.234224:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234227:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234230:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234234:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234251:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765032.234261:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067934080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234268:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234271:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765032.234272:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234274:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234277:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234289:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234293:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765032.234301:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067934208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765032.234314:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765032.234317:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765032.234318:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765032.234322:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2500 00000100:00080000:2.0:1789765032.234325:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765032.234328:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765032.234331:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765032.234338:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067934336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765032.234627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067933568:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765032.234683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.234732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061de680 x1876702067933568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.234748:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067933568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.234757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.234760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.234764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067933568:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765032.234771:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765032.234791:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.234830:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061dfb80 x1876702067933696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.234842:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067933696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.234856:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.234859:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.234862:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765032.234868:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765032.234886:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.234898:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061df800 x1876702067933824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.234909:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067933824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.234917:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.234920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.234922:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765032.234927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765032.234947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.234958:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061dca80 x1876702067933952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.234969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067933952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.234976:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.234979:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.234982:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067933952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765032.234986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765032.235004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.235015:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061dd180 x1876702067934080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.235027:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067934080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.235034:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.235036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.235040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765032.235045:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067934208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765032.235065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.235075:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061dce00 x1876702067934208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.235085:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067934208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.235092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.235095:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.235098:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067934208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765032.235104:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067934336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765032.235120:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765032.235130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765032/real 1789765032] req@ffff9360061dc000 x1876702067934336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765032.235140:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067934336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765057 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765032.235148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765032.235151:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765032.235154:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067934336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765033.002132:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765034.026559:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765035.050026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765036.075132:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765037.098008:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765037.353979:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.353989:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765037.353993:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.354002:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.354012:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.354018:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765037.354053:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067934464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765037.354131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067934464:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765037.354198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765037.354227:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.354231:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765037.354233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.354237:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.354242:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.354246:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.354255:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765037.354266:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067934592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765037.354363:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020daf480 x1876702067934464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.354377:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067934464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.354387:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.354390:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.354393:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067934464:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765037.354413:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765037.354436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765037.354446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020daed80 x1876702067934592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.354456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067934592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.354469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.354472:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.354475:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934592:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765037.354557:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.354561:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765037.354563:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.354567:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.354571:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.354575:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.354580:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765037.354591:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067934720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765037.354633:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765037.354654:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765037.354673:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.354676:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765037.354677:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.354681:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.354685:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.354688:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.354693:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765037.354703:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067934848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765037.354736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020dae680 x1876702067934720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.354747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067934720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.354754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.354757:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.354760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765037.354770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765037.354787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765037.354819:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020daed80 x1876702067934848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.354833:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067934848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.354840:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.354842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.354846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934848:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765037.355032:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.355036:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765037.355038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.355042:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.355045:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.355048:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.355053:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765037.355064:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067934976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765037.355072:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.355075:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765037.355077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.355079:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.355082:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.355086:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.355089:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765037.355096:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067935104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765037.355109:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765037.355112:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765037.355113:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765037.355116:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2505 00000100:00080000:0.0:1789765037.355119:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765037.355122:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765037.355125:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765037.355132:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067935232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765037.355187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765037.355210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765037.355221:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020daf480 x1876702067934976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.355232:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067934976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.355239:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.355242:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.355245:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067934976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765037.355250:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067935104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765037.355269:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765037.355280:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020dad500 x1876702067935104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.355290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067935104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.355297:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.355299:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.355302:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067935104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765037.355306:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067935232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765037.355321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765037.355330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765037/real 1789765037] req@ffff936020dad880 x1876702067935232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765037.355340:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067935232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765062 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765037.355347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765037.355350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765037.355353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067935232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765038.121988:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765039.146190:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765040.170051:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765041.196348:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765042.219237:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765042.477410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765042.477424:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477434:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477444:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477451:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765042.477488:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067935360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477526:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765042.477528:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477532:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477536:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477540:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477549:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765042.477561:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067935488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765042.477575:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477578:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477581:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765042.477598:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067935616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477610:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765042.477611:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477615:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477618:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477621:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477625:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765042.477636:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067935744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765042.477649:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477652:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477655:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477658:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477662:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765042.477671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067935872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765042.477683:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477687:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477690:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477698:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765042.477707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067936000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765042.477716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765042.477719:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765042.477720:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765042.477724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2510 00000100:00080000:2.0:1789765042.477727:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765042.477731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765042.477735:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765042.477745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067936128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765042.479005:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067935360:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765042.479066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061dd500 x1876702067935360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479199:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067935360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479209:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067935360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765042.479224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765042.479263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061df480 x1876702067935488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479288:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067935488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479302:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479309:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765042.479315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765042.479336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061df100 x1876702067935616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067935616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479366:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479369:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479371:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765042.479377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765042.479397:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061dd880 x1876702067935744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067935744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479427:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765042.479439:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765042.479458:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061ddc00 x1876702067935872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067935872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479489:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479495:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067935872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765042.479501:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765042.479520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765042.479531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061de300 x1876702067936000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765042.479543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067936000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765042.479550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765042.479554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765042.479557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765042.481874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765042.481960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765042.481990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765042/real 1789765042] req@ffff9360061ddf80 x1876702067936128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765042.482011:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067936128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765042.482022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765042.482027:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765042.482031:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765043.245405:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765044.267240:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765045.294343:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765046.315162:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765047.338427:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765047.594052:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594062:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765047.594066:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594076:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594087:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594107:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765047.594142:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067936256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765047.594178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594182:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594186:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594189:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594198:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765047.594208:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067936384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594218:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594222:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765047.594223:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594227:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594230:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594233:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594237:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765047.594245:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067936512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594260:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594264:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765047.594265:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594269:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594271:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594275:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594279:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765047.594287:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067936640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594296:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594299:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765047.594300:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594303:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594307:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594310:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594314:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765047.594322:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067936768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594331:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594334:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765047.594335:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594339:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594342:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594346:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594350:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765047.594358:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067936896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765047.594366:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765047.594369:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765047.594371:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765047.594374:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2515 00000100:00080000:1.0:1789765047.594377:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765047.594380:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765047.594384:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765047.594391:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067937024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765047.594483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067936256:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765047.594554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594593:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef6300 x1876702067936256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.594621:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067936256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.594631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.594635:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.594639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067936256:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765047.594646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765047.594667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594678:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef6d80 x1876702067936384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.594690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067936384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.594698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.594701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.594704:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765047.594709:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765047.594727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594737:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef4e00 x1876702067936512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.594748:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067936512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.594756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.594759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.594762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765047.594768:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765047.594785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594795:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef7100 x1876702067936640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.594831:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067936640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.594839:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.594842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.594845:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765047.594851:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765047.594869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594879:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef4380 x1876702067936768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.594890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067936768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.594898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.594901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.594904:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067936768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765047.594909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765047.594926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.594936:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef4000 x1876702067936896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.595118:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067936896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.595127:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.595130:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.595133:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067936896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765047.595140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765047.595160:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765047.595170:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765047/real 1789765047] req@ffff936010ef5c00 x1876702067937024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765047.595182:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067937024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765047.595190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765047.595193:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765047.595196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765048.361973:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765049.386026:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765050.411151:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765051.434092:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765052.469962:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765052.714921:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.714929:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765052.714932:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.714941:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.714950:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.714957:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765052.714989:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067937152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.715017:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.715021:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765052.715023:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.715028:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.715032:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.715036:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.715044:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765052.716027:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067937152:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765052.716935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.716976:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dac000 x1876702067937152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.716996:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067937152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.717006:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.717010:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.717014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067937152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765052.717598:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067937280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.717616:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.717620:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765052.717622:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.717627:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.717632:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.717638:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.717646:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765052.717663:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067937408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.717672:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.717675:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765052.717676:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.717680:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.717683:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.717686:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.717690:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765052.717699:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067937536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.717715:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.717718:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765052.717719:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.717722:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.717726:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.717729:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.717734:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765052.717744:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067937664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.717753:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.717756:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765052.717758:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.717761:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.717765:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.717769:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.717773:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765052.717783:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067937792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765052.717791:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765052.717794:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765052.717818:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765052.717823:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2520 00000100:00080000:0.0:1789765052.717827:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765052.717830:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765052.717834:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765052.717844:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067937920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765052.718398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765052.718438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718456:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dafb80 x1876702067937280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718468:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067937280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718477:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718480:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718484:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765052.718490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765052.718531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718542:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dac700 x1876702067937408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718553:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067937408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718561:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718564:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765052.718571:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765052.718591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020daf100 x1876702067937536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067937536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718617:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765052.718626:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765052.718643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718653:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dac380 x1876702067937664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718663:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067937664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718671:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718674:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718676:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067937664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765052.718682:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765052.718701:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dad180 x1876702067937792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718722:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067937792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765052.718746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765052.718764:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765052.718774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765052/real 1789765052] req@ffff936020dae300 x1876702067937920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765052.718784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067937920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765052.718792:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765052.718795:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765052.718826:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067937920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765053.482032:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765054.508598:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765055.530087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765056.554298:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765057.578364:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765057.834584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765057.834600:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834609:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834623:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834630:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765057.834667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067938048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765057.834706:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834710:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834714:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834717:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765057.834738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067938176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834747:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765057.834751:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834762:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765057.834777:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067938304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765057.834791:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834794:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834834:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834838:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834842:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765057.834854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067938432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765057.834867:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834870:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834873:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834876:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765057.834888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067938560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834900:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765057.834902:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834905:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834909:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834912:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834916:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765057.834925:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067938688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765057.834933:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765057.834935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765057.834937:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765057.834940:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2525 00000100:00080000:3.0:1789765057.834943:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765057.834945:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765057.834949:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765057.834959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067938816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765057.835054:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067938048:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765057.835111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa7b80 x1876702067938048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835162:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067938048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835171:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835176:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835180:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067938048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765057.835187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765057.835208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa6300 x1876702067938176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067938176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835254:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765057.835260:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765057.835279:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835290:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa4000 x1876702067938304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835302:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067938304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835310:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835312:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765057.835321:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765057.835338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835347:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa7800 x1876702067938432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835357:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067938432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835365:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835367:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765057.835375:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765057.835389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa5c00 x1876702067938560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835408:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067938560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835417:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835420:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067938560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765057.835425:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067938688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765057.835442:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835452:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa5880 x1876702067938688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835462:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067938688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835485:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835488:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067938688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765057.835494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067938816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765057.835512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765057.835521:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765057/real 1789765057] req@ffff936010aa4a80 x1876702067938816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765057.835532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067938816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765057.835539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765057.835542:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765057.835545:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067938816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765058.602523:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765059.626316:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765060.650597:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765061.674094:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765062.698012:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765062.953954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.953963:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765062.953967:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.953976:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.953998:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954006:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765062.954043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067938944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765062.954064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.954068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765062.954070:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954074:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954079:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789765062.954080:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067938944:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765062.954085:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765062.954096:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765062.954110:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067939072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765062.954120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.954124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765062.954125:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954129:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954133:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954136:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765062.954141:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765062.954151:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067939200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765062.954161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.954164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765062.954165:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954169:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954173:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954177:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765062.954181:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765062.954192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067939328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765062.954202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765062.954203:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939072:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765062.954208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765062.954212:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954216:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954220:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954223:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765062.954229:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765062.954240:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765062.954243:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067939456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765062.954255:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.954259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765062.954260:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954264:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954269:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954273:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765062.954277:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939328:192.168.203.137@tcp:8:kworker.0 00000008:00100000:3.0:1789765062.954278:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765062.954295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067939584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765062.954309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939456:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765062.954312:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765062.954317:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765062.954319:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765062.954324:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2530 00000100:00080000:3.0:1789765062.954328:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765062.954333:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765062.954337:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067939584:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789765062.954338:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765062.954355:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067939712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765062.954374:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067939712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765063.723385:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765064.775483:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765065.782408:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765066.026068:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765066.026118:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765066.026133:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765066.031730:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa4380 x1876702067938944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765087 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.031760:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067938944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765087 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.031771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.031776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.031780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067938944:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765066.031901:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765066.031922:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765066.031929:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765066.031936:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765066.031942:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765066.031949:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765066.032004:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa4e00 x1876702067939712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032019:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067939712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032027:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067939712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765066.032040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa5f80 x1876702067939584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032050:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067939584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067939584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765066.032069:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa7480 x1876702067939328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067939328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032095:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032098:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032100:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765066.032106:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa6680 x1876702067939456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032116:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067939456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032122:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032125:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032128:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765066.032133:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa5180 x1876702067939072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067939072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032155:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765066.032308:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765062/real 1789765066] req@ffff936010aa7100 x1876702067939200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765066.032321:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067939200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765066.032329:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765066.032332:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765066.032336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939200:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765066.794044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765067.822629:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765068.079453:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079463:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765068.079467:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079477:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079488:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079495:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765068.079532:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067939840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079574:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079577:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765068.079579:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079583:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079586:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079590:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079598:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765068.079609:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067939968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079619:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079622:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765068.079623:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079627:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079629:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079632:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079635:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765068.079643:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067940096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079652:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079655:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765068.079656:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079661:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079663:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079666:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079670:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765068.079679:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067940224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079687:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079689:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765068.079691:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079694:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079698:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079701:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079705:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765068.079715:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067940352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079723:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079726:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765068.079727:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079730:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079733:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079736:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079739:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765068.079746:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067940480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765068.079754:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765068.079757:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765068.079759:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765068.079762:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2535 00000100:00080000:1.0:1789765068.079764:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765068.079767:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765068.079770:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765068.079788:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067940608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765068.079902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067939840:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765068.079972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.080005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef5880 x1876702067939840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.080021:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067939840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.080030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.080034:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.080038:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067939840:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765068.080044:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765068.080070:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.080081:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef4a80 x1876702067939968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.080104:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067939968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.080112:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.080115:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.080118:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067939968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765068.080123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765068.080138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.081424:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef5180 x1876702067940096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.081448:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067940096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.081458:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.081461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.081465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765068.081473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765068.083145:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.083176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef4700 x1876702067940224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.083194:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067940224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.083211:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.083215:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.083219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765068.083228:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765068.083258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.083268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef5c00 x1876702067940352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.083279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067940352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.083286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.083289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.083292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765068.083298:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067940480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765068.083314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.083327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef4000 x1876702067940480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.083337:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067940480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.083344:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.083347:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.083350:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067940480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765068.083357:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067940608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765068.083376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765068.083386:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765068/real 1789765068] req@ffff936010ef4380 x1876702067940608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765068.083406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067940608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765068.083420:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765068.083423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765068.083426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067940608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765068.842366:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765069.869059:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765070.890967:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765071.914409:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765072.941123:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765073.193980:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.193990:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765073.193994:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194003:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:0.0:1789765073.194013:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765073.194019:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765073.194055:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067940736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765073.194085:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194088:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765073.194090:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194093:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:0.0:1789765073.194096:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765073.194099:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765073.194115:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765073.194125:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067940864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765073.194135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194137:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765073.194139:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194142:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:0.0:1789765073.194145:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765073.194147:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765073.194151:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765073.194159:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067940992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765073.194189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067940736:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765073.194229:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194233:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765073.194235:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194240:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:0.0:1789765073.194245:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789765073.194249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765073.194252:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765073.194259:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765073.194272:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067941120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765073.194281:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194284:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765073.194287:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194290:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:0.0:1789765073.194295:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789765073.194295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020daf800 x1876702067940736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765073.194301:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765073.194309:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765073.194320:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067940736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765073.194324:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067941248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765073.194333:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194339:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765073.194343:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194351:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2541 00000100:00080000:2.0:1789765073.194353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765073.194355:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765073.194362:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789765073.194362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:0.0:1789765073.194371:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765073.194372:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067940736:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765073.194383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765073.194387:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067941376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765073.194398:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765073.194401:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765073.194403:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765073.194406:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2541 00000800:00000100:2.0:1789765073.194410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765073.194411:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765073.194418:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765073.194422:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765073.194428:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020dadc00 x1876702067940864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765073.194434:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067941504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765073.194452:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067940864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194466:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765073.194490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765073.194505:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765073.194526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020dadf80 x1876702067940992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765073.194538:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067940992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194546:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194549:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194552:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067940992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765073.194557:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765073.194573:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765073.194581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020daea00 x1876702067941120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765073.194591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067941120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194605:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765073.194610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765073.194623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765073.194631:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020dace00 x1876702067941248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765073.194641:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067941248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765073.194658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067941376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765073.194672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765073.194680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020dae300 x1876702067941376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765073.194689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067941376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194697:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067941376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765073.194708:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067941504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765073.194721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765073.194728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765073/real 1789765073] req@ffff936020dad180 x1876702067941504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765073.194739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067941504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765073.194746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765073.194748:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765073.194751:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067941504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765073.962331:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765074.986067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765076.010136:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765077.033959:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765078.058291:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765078.316961:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.316971:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765078.316975:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.316984:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.316994:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317001:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765078.317040:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067941632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317071:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317075:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765078.317077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317081:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317085:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317089:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317098:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765078.317361:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067941760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317373:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317377:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765078.317379:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317384:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317388:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317393:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317399:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765078.317411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067941888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317420:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317423:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765078.317425:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317428:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317432:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317435:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317439:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765078.317449:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067942016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317458:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317461:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765078.317462:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317465:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317468:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317471:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317476:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765078.317485:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067942144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317495:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317498:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765078.317500:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317503:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317506:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317510:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317514:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765078.317522:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067942272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765078.317531:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765078.317534:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765078.317536:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765078.317539:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2546 00000100:00080000:1.0:1789765078.317542:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765078.317545:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765078.317549:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765078.317557:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067942400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765078.317726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067941632:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765078.317784:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.317856:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef7100 x1876702067941632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.317882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067941632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.317892:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.317895:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.317899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067941632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765078.317905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765078.317927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.317940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef4e00 x1876702067941760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.317951:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067941760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.317959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.317961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.317964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765078.317970:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765078.317988:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.317999:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef6d80 x1876702067941888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.318008:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067941888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.318015:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.318018:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.318021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067941888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765078.318026:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765078.318046:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.318057:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef6300 x1876702067942016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.318068:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067942016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.318076:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.318090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.318093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765078.318099:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765078.318192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.318204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef7b80 x1876702067942144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.318215:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067942144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.318223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.318226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.318229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765078.318235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067942272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765078.318256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.318267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef7480 x1876702067942272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.318277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067942272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.318284:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.318287:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.318291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067942272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765078.318297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067942400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765078.318316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765078.318327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765078/real 1789765078] req@ffff936010ef6680 x1876702067942400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765078.318337:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067942400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765103 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765078.318344:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765078.318347:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765078.318350:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067942400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765079.082006:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765080.106042:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765081.130300:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765082.156308:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765083.178140:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765083.435371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435381:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765083.435385:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435395:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435406:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435413:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765083.435449:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067942528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765083.435487:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435491:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435494:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435498:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435508:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765083.435519:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067942656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765083.435533:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435536:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435540:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435543:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435548:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765083.435560:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067942784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765083.435572:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435575:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435579:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435582:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435587:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765083.435596:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067942912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765083.435609:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435613:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435616:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435619:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435623:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765083.435633:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067943040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765083.435646:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435652:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435655:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435659:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765083.435668:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067943168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765083.435675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765083.435678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765083.435679:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765083.435682:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2551 00000100:00080000:2.0:1789765083.435685:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765083.435688:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765083.435693:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765083.435701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067943296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765083.435781:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067942528:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765083.435865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.435905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061dc380 x1876702067942528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.435922:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067942528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.435933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.435937:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.435940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067942528:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765083.435947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765083.435970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.435983:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061dc700 x1876702067942656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.435995:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067942656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436002:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436005:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765083.436014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765083.436034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.436045:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061ddf80 x1876702067942784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.436056:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067942784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436064:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436067:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765083.436077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765083.436097:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.436108:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061de300 x1876702067942912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.436118:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067942912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436126:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436129:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436131:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067942912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765083.436136:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765083.436155:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.436166:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061ddc00 x1876702067943040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.436177:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067943040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436184:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765083.436195:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067943168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765083.436215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.436226:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061dd880 x1876702067943168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.436237:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067943168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436245:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436250:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067943168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765083.436255:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067943296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765083.436274:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765083.436284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765083/real 1789765083] req@ffff9360061df100 x1876702067943296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765083.436294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067943296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765083.436301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765083.436304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765083.436306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067943296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765084.202308:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765085.229559:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765086.250216:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765087.274065:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765088.298062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765088.553924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.553935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765088.553938:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.553947:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.553956:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.553962:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765088.553994:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067943424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765088.554022:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554026:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554032:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554040:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765088.554049:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067943552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765088.554062:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554071:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554074:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765088.554082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067943680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765088.554094:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554097:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554099:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554102:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554105:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765088.554112:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067943808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765088.554123:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554126:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554128:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554131:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554135:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765088.554142:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067943936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765088.554162:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554164:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554167:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554169:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554172:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765088.554179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067944064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765088.554186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765088.554189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765088.554190:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765088.554193:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2556 00000100:00080000:2.0:1789765088.554195:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765088.554198:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765088.554201:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765088.554207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067944192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765088.554394:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067943424:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765088.554500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.554535:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061df480 x1876702067943424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.554550:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067943424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.554560:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.554563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.554567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067943424:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765088.554573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765088.554591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.554601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061dd500 x1876702067943552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.554612:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067943552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.554619:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.554622:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.554625:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765088.554629:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765088.554643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.554650:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061dc000 x1876702067943680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.554660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067943680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.554667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.554670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.554673:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765088.554678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765088.554692:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.554700:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061dce00 x1876702067943808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.554709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067943808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.554716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.554718:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.554721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765088.554726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765088.554739:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.554746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061dd180 x1876702067943936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.554756:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067943936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.554763:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.554765:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.554768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067943936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765088.554772:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765088.554785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.555369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061dca80 x1876702067944064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.555386:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067944064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.555394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.555397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.555400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765088.555406:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765088.555429:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765088.555440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765088/real 1789765088] req@ffff9360061df800 x1876702067944192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765088.555507:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067944192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765088.555515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765088.555517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765088.555520:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765089.324021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765090.346394:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765091.370274:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765092.393947:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765093.418182:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765093.677198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765093.677213:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677222:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677237:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677244:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765093.677295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067944320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677335:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765093.677337:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677342:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677345:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677349:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677358:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765093.677369:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067944448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677379:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677382:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765093.677383:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677387:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677390:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677393:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677397:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765093.677406:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067944576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677432:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677435:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765093.677436:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677439:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677442:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677446:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677450:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765093.677461:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067944704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677469:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765093.677473:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677484:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677488:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677491:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677496:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765093.677505:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067944832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677514:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765093.677519:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677522:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677525:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677528:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677532:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765093.677541:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067944960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765093.677549:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765093.677552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765093.677553:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765093.677556:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2561 00000100:00080000:3.0:1789765093.677558:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765093.677561:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765093.677565:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765093.677572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067945088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765093.677639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067944320:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765093.677706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.677751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa4700 x1876702067944320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.677787:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067944320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.677796:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.677820:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.677824:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067944320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765093.677831:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765093.677854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.677867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa6a00 x1876702067944448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.677878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067944448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.677885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.677888:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.677891:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765093.677896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765093.677914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.677925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa5500 x1876702067944576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.677935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067944576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.677947:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.677950:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.677952:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765093.677957:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765093.677989:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.678016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa7100 x1876702067944704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.678030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067944704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.678038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.678041:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.678045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765093.678051:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765093.678069:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.678080:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa5180 x1876702067944832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.678091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067944832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.678097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.678100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.678103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067944832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765093.678108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765093.678128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.678139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa6680 x1876702067944960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.678149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067944960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.678155:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.678158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.678161:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067944960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765093.678166:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765093.678184:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765093.678195:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765093/real 1789765093] req@ffff936010aa7480 x1876702067945088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765093.678206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067945088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765093.678212:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765093.678215:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765093.678218:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765094.445687:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765095.466790:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765096.489983:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765097.514459:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765098.537996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765098.794412:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.794422:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765098.794426:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.794436:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.794445:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.794452:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765098.794488:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067945216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765098.794566:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067945216:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765098.794627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765098.794657:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.794662:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765098.794664:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.794668:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.794673:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.794676:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765098.794684:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765098.794698:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067945344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765098.796284:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.796293:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765098.796296:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.796304:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.796311:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.796317:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765098.796326:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765098.796334:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dac380 x1876702067945216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765098.796359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067945216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765098.796362:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067945472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765098.796379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765098.796383:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765098.796387:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765098.796389:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765098.796392:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789765098.796394:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067945216:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765098.796400:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.796406:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.796411:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765098.796415:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765098.796426:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765098.796427:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067945600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765098.796441:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.796444:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765098.796445:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.796449:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.796453:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.796456:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765098.796460:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765098.796471:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067945728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789765098.796476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765098.796486:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.796490:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765098.796491:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.796496:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00100000:2.0:1789765098.796498:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020daf100 x1876702067945344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765098.796507:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765098.796515:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765098.796519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067945344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789765098.796522:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789765098.796535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789765098.796540:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067945856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765098.796541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.796548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765098.796559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945472:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765098.796561:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765098.796571:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765098.796573:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765098.796579:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2566 00000100:00080000:0.0:1789765098.796583:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789765098.796586:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765098.796590:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765098.796595:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765098.796601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dac700 x1876702067945472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765098.796610:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067945984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765098.796617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067945472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765098.796661:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765098.796666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.796669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765098.796685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765098.796707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765098.796735:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dafb80 x1876702067945600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765098.796747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067945600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765098.796755:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765098.796758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.796761:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765098.796766:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765098.796785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765098.797047:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dac000 x1876702067945728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765098.797061:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067945728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765098.797078:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765098.797081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.797085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067945728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765098.797092:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765098.797113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765098.797125:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dad880 x1876702067945856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765098.797136:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067945856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765098.797143:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765098.797146:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.797149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765098.797154:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765098.797173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765098.797183:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765098/real 1789765098] req@ffff936020dad500 x1876702067945984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765098.797195:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067945984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765098.797202:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765098.797205:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765098.797207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067945984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765099.562102:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765100.586004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765101.609998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765102.634599:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765103.657987:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765103.913896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.913905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765103.913909:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.913918:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.913928:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.913935:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765103.914000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067946112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765103.914037:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914041:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914045:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914049:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914057:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765103.914070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067946240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765103.914084:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914088:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914092:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914096:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914100:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765103.914111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067946368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765103.914124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914134:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914138:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765103.914148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067946496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765103.914161:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914165:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914168:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914171:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765103.914185:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067946624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765103.914203:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914206:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914209:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914213:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914217:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765103.914226:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067946752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765103.914233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765103.914236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765103.914238:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765103.914242:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2571 00000100:00080000:0.0:1789765103.914244:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765103.914247:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765103.914262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765103.914272:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067946880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765103.914913:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067946112:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765103.914972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020daf480 x1876702067946112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067946112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915065:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067946112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765103.915072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765103.915095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915108:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020daed80 x1876702067946240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915119:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067946240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915127:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765103.915140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765103.915159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915171:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020dae680 x1876702067946368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915182:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067946368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915192:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765103.915202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765103.915221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915232:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020dad500 x1876702067946496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915243:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067946496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915263:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915271:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765103.915277:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765103.915297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915427:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020dad880 x1876702067946624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915440:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067946624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915448:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915455:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067946624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765103.915460:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067946752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765103.915482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020dac000 x1876702067946752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915505:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067946752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915519:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067946752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765103.915524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067946880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765103.915543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765103.915553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765103/real 1789765103] req@ffff936020dafb80 x1876702067946880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765103.915563:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067946880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765103.915571:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765103.915574:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765103.915576:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067946880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765104.682324:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765105.708718:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765106.730059:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765107.754821:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765108.778092:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765109.033947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.033958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765109.033963:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.033972:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.033983:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.033990:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765109.034024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067947008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765109.034065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.034069:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765109.034070:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.034073:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.034077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034080:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765109.034096:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765109.034106:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067947136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765109.034115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.034118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765109.034120:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.034123:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.034125:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034128:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765109.034132:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765109.034140:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067947264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765109.034154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765109.034201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067947008:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765109.034257:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.034295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020dac700 x1876702067947008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765109.034298:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765109.034301:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789765109.034317:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067947008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.034333:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.034337:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.034340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067947008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765109.034347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765109.034367:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.034378:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020daf100 x1876702067947136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765109.034390:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067947136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.034397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.034400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.034404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765109.034409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947264:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765109.034425:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2576 00000800:00000100:1.0:1789765109.034430:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765109.034433:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034439:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765109.034445:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020dac380 x1876702067947264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789765109.034446:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765109.034466:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067947392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765109.034466:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067947264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765109.034496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.034499:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765109.034501:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.034505:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.034507:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034510:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765109.034514:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765109.034521:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067947520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765109.034530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.034533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765109.034534:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.034537:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.034540:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034543:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765109.034547:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765109.034555:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067947648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765109.034563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765109.034566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765109.034567:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765109.034570:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2576 00000100:00080000:0.0:1789765109.034572:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765109.034575:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765109.034578:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765109.034585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067947776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765109.036265:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.036273:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.036277:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765109.036314:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765109.036357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.036375:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020dad180 x1876702067947392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765109.036390:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067947392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.036398:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.036402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.036405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765109.036413:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765109.036433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.036444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020dae300 x1876702067947520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765109.036455:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067947520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.036462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.036465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.036468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067947520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765109.036474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067947648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765109.036589:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.036601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020dace00 x1876702067947648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765109.036614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067947648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.036622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.036626:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.036629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067947648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765109.036635:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067947776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765109.036668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765109.036680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765109/real 1789765109] req@ffff936020daea00 x1876702067947776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765109.036692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067947776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765109.036700:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765109.036703:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765109.036707:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067947776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765109.802101:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765110.826671:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765111.850618:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765112.877594:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765113.898014:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765114.156326:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156336:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765114.156340:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156348:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156359:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156366:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765114.156401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067947904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156435:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156439:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765114.156441:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156445:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156449:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156452:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156461:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765114.156471:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067948032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156479:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156482:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765114.156484:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156487:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156489:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156492:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156496:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765114.156504:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067948160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156512:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156514:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765114.156516:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156519:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156522:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156526:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156529:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765114.156549:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067948288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156558:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156561:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765114.156562:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156566:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156569:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156571:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156575:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765114.156587:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067948416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156595:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156598:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765114.156599:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156602:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156614:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156617:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156621:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765114.156631:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067948544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765114.156644:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765114.156647:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765114.156648:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765114.156652:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2581 00000100:00080000:1.0:1789765114.156655:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765114.156658:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765114.156662:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765114.156671:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067948672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765114.156829:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067947904:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765114.156890:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.156944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef5f80 x1876702067947904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.156961:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067947904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.156971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.156975:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.156978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067947904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765114.156986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765114.157008:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157021:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef6a00 x1876702067948032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157034:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067948032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157041:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157047:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765114.157052:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765114.157072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157083:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef5500 x1876702067948160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157093:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067948160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157106:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157109:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157113:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765114.157118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765114.157140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef7800 x1876702067948288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157163:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067948288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157171:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157175:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157178:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765114.157183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765114.157203:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157269:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef6680 x1876702067948416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157283:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067948416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157291:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157294:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157297:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765114.157303:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067948544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765114.157323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157335:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef7480 x1876702067948544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157345:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067948544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157353:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157359:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067948544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765114.157364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067948672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765114.157384:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765114.157395:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765114/real 1789765114] req@ffff936010ef7b80 x1876702067948672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765114.157406:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067948672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765114.157413:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765114.157416:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765114.157419:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067948672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765114.922073:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765115.948352:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765116.970025:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765117.996957:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765119.018496:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765119.274640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765119.274653:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274662:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274671:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274678:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765119.274710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067948800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765119.274741:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274744:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274750:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274758:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765119.274766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067948928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765119.274780:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274785:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274788:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274791:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765119.274821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067949056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274830:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274833:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765119.274835:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274838:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274841:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274845:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274849:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765119.274860:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067949184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274870:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765119.274872:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274875:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274878:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274881:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765119.274894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067949312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274905:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765119.274906:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274909:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274912:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274915:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274919:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765119.274928:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067949440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765119.274936:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765119.274939:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765119.274940:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765119.274944:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2587 00000100:00080000:0.0:1789765119.274947:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765119.274949:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765119.274953:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765119.274962:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067949568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765119.275040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067948800:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765119.275093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020dadf80 x1876702067948800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067948800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275154:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067948800:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765119.275168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765119.275190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275202:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020dadc00 x1876702067948928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275214:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067948928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275241:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067948928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765119.275246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765119.275265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020daf800 x1876702067949056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275287:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067949056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765119.275311:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765119.275330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275341:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020daea00 x1876702067949184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275352:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067949184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765119.275370:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765119.275388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020dace00 x1876702067949312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275409:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067949312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275419:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275423:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765119.275428:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067949440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765119.275446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275456:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020dae300 x1876702067949440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275467:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067949440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275477:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275480:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067949440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765119.275486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067949568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765119.275504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765119.275514:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765119/real 1789765119] req@ffff936020dad180 x1876702067949568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765119.275525:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067949568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765144 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765119.275533:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765119.275535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765119.275538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067949568:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765120.043153:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765121.068645:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765122.092677:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765123.114250:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765124.139066:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765124.394419:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394429:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765124.394433:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394443:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394455:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394463:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765124.394498:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067949696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765124.394541:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394546:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394553:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394561:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765124.394574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067949824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765124.394586:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394590:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394594:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394602:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765124.394613:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067949952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394623:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394626:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765124.394628:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394631:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394634:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394637:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765124.394651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067950080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394661:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765124.394663:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394666:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394669:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394673:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394677:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765124.394687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067950208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765124.394701:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394704:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394707:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394711:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394715:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765124.394723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067950336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765124.394730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765124.394734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765124.394746:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765124.394751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2592 00000100:00080000:0.0:1789765124.394755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765124.394758:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765124.394763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765124.394775:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067950464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765124.395861:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067949696:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765124.395964:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.397949:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dac380 x1876702067949696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.397988:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067949696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398003:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067949696:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765124.398026:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765124.398094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020daf100 x1876702067949824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067949824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398142:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398145:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398149:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765124.398156:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765124.398180:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398193:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dac700 x1876702067949952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067949952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398226:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067949952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765124.398233:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765124.398255:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dafb80 x1876702067950080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067950080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398287:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398293:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765124.398299:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765124.398320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398331:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dac000 x1876702067950208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067950208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765124.398362:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067950336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765124.398383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398394:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dad880 x1876702067950336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398405:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067950336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398411:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398414:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398418:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067950336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765124.398423:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067950464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765124.398443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765124.398454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765124/real 1789765124] req@ffff936020dad500 x1876702067950464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765124.398465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067950464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765149 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765124.398472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765124.398475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765124.398478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067950464:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765125.162887:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765126.187274:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765127.210007:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765128.235641:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765129.258154:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765129.513906:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.513917:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765129.513920:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.513930:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.513940:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.513947:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765129.513981:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067950592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765129.514010:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514013:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765129.514015:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514018:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514022:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514025:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514035:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765129.514047:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067950720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765129.514057:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514060:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765129.514061:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514064:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514067:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514071:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514075:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765129.514085:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067950848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765129.514091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067950592:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765129.514094:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514096:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765129.514098:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514101:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514104:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514107:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514111:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765129.514119:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067950976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765129.514126:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514128:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765129.514129:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514132:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514137:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514140:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514143:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765129.514150:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067951104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765129.514158:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514161:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765129.514162:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514165:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514168:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514170:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514175:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765129.514182:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765129.514183:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067951232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765129.514190:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765129.514193:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765129.514194:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765129.514197:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2597 00000100:00080000:0.0:1789765129.514200:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765129.514203:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765129.514206:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765129.514212:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765129.514214:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067951360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765129.514231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765129.514263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765129.514283:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067951232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765129.514306:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067951360:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765130.282124:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765131.306741:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765132.330977:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789765132.587410:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789765132.587458:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789765132.588186:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588232:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588246:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588255:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588262:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588275:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765132.588283:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:2.0:1789765132.590360:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020dafb80 x1876702067951360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590392:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067951360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590409:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067951360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765132.590422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020daf480 x1876702067950848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590434:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067950848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765132.590453:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020dad500 x1876702067950976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067950976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590483:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590486:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590490:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765132.590496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020dad880 x1876702067951104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590509:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067951104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765132.590531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020dac000 x1876702067951232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067951232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067951232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765132.590563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020dae680 x1876702067950592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067950592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590584:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590590:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067950592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765132.590595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765129/real 1789765132] req@ffff936020daed80 x1876702067950720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765132.590604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067950720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765132.590610:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765132.590613:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765132.590616:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067950720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765133.354860:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765134.378150:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765134.634275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765134.634288:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634297:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634306:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634313:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765134.634347:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067951488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634382:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765134.634388:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634392:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634396:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634399:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634408:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765134.634421:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067951616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634433:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765134.634434:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634438:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634441:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634444:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634448:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765134.634468:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067951744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765134.634480:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634483:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634486:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634490:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634494:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765134.634503:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067951872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765134.634515:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634518:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634520:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634523:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634526:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765134.634534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067952000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634548:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765134.634553:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634557:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634563:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634568:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765134.634576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067952128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765134.634584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765134.634587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765134.634588:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765134.634592:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2602 00000100:00080000:3.0:1789765134.634595:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765134.634598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765134.634603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765134.634611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067952256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765134.635262:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067951488:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765134.635319:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa5f80 x1876702067951488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067951488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635401:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067951488:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765134.635412:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765134.635434:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa4e00 x1876702067951616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635470:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067951616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635478:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765134.635489:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765134.635509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635519:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa4380 x1876702067951744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067951744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635645:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765134.635655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765134.635678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635688:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa4a80 x1876702067951872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067951872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635705:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635708:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067951872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765134.635717:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765134.635736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa5880 x1876702067952000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635756:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067952000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635764:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635767:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635770:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765134.635776:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067952128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765134.635832:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa5c00 x1876702067952128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635860:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067952128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635866:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.635868:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.635871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067952128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765134.635876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067952256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765134.635901:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765134.635911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765134/real 1789765134] req@ffff936010aa7800 x1876702067952256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765134.635922:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067952256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765134.635928:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765134.636430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765134.636436:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067952256:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765135.402199:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765136.426748:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765137.450793:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765138.474373:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765139.499473:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765139.754335:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.754344:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765139.754347:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.754356:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.754366:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.754372:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765139.754405:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067952384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765139.754445:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.754449:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765139.754451:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.754454:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.754458:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.754461:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.754471:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765139.754482:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067952512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765139.754491:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.754494:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765139.754496:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.754500:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.754503:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.754506:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.754509:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765139.754519:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067952640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765139.754619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067952384:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765139.754676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.754714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef6300 x1876702067952384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.754731:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067952384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.754746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.754749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.754753:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067952384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765139.754760:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765139.754777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.754788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef6d80 x1876702067952512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.754822:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067952512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.754831:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.754834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.754838:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952512:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765139.755324:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.755328:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765139.755330:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.755334:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.755338:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.755342:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.755348:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765139.755360:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067952768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765139.755369:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.755372:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765139.755374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.755377:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.755379:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.755382:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.755386:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765139.755394:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067952896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765139.755401:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.755404:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765139.755406:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.755409:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.755411:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.755414:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.755427:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765139.755435:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067953024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765139.755444:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765139.755446:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765139.755448:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765139.755451:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2607 00000100:00080000:1.0:1789765139.755454:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765139.755457:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765139.755461:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765139.755468:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067953152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765139.755516:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765139.755551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.755563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef4e00 x1876702067952640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.755575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067952640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.755582:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.755584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.755587:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765139.755592:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765139.755609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.755620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef7100 x1876702067952768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.755630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067952768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.755649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.755653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.755655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765139.755660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765139.755677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.755686:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef4380 x1876702067952896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.755697:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067952896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.755704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.755707:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.755710:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067952896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765139.755715:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765139.755730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.755738:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef4000 x1876702067953024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.755748:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067953024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.755756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.755758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.755761:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765139.755766:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765139.755780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765139.755789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765139/real 1789765139] req@ffff936010ef5c00 x1876702067953152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765139.755824:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067953152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765139.755831:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765139.755834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765139.755837:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953152:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765140.522049:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765141.546116:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765142.570135:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765143.596647:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765144.618968:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765144.884666:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.884676:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765144.884679:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.884689:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.884699:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.884705:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765144.884741:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067953280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765144.884831:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067953280:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765144.884884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765144.884944:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.884948:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765144.884950:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.884954:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.884958:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.884962:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.884970:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765144.884984:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067953408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765144.885056:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020dac700 x1876702067953280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.885086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067953280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.885291:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.885295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.885300:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067953280:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765144.885321:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765144.885343:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765144.885355:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020daf100 x1876702067953408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.885367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067953408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.885391:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.885395:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.885398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953408:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765144.885714:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.885719:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765144.885720:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.885724:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.885729:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.885733:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.885739:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765144.885751:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067953536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765144.885791:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765144.885828:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765144.885858:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.885862:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765144.885864:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.885868:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.885871:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.885874:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.885879:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765144.885890:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067953664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765144.885924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020dac380 x1876702067953536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.885936:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067953536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.885944:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.885947:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.885951:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765144.885961:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765144.885980:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765144.885990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020dad180 x1876702067953664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.886001:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067953664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.886008:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.886011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.886014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953664:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765144.886056:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.886279:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765144.886281:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.886286:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.886289:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.886293:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.886298:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765144.886310:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067953792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765144.886320:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.886323:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765144.886325:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.886328:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.886330:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.886334:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.886338:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765144.886345:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067953920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765144.886353:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765144.886355:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765144.886357:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765144.886360:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2612 00000100:00080000:0.0:1789765144.886362:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765144.886365:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765144.886368:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765144.886396:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067954048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765144.886477:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765144.886598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765144.886612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020dae300 x1876702067953792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.886624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067953792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.886632:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.886635:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.886638:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067953792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765144.886644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765144.886662:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765144.886671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020dace00 x1876702067953920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.886682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067953920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.886691:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.886694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.886697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067953920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765144.886701:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765144.886716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765144.886724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765144/real 1789765144] req@ffff936020daea00 x1876702067954048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765144.886734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067954048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765144.886742:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765144.886744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765144.886748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954048:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765145.642435:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765146.668840:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765147.690405:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765148.715248:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765149.738413:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765149.993895:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.993904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765149.993909:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.993918:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.993928:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.993935:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765149.993968:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067954176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765149.994009:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994012:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994015:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994018:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994027:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765149.994037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067954304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994046:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765149.994050:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994054:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994067:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994070:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994074:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765149.994082:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067954432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765149.994095:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994099:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994101:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994104:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994108:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765149.994116:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067954560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765149.994205:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994209:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994212:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994216:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994220:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765149.994231:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067954688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765149.994245:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994248:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994251:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994254:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994257:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765149.994264:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067954816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765149.994272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765149.994275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765149.994277:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765149.994280:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2617 00000100:00080000:0.0:1789765149.994282:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765149.994285:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765149.994288:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765149.994295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067954944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765149.994399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067954176:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765149.994459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994505:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020daf800 x1876702067954176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067954176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067954176:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765149.994549:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765149.994570:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020dadc00 x1876702067954304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067954304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765149.994608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765149.994626:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994637:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020dadf80 x1876702067954432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067954432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765149.994667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765149.994685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020daea00 x1876702067954560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067954560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994710:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765149.994720:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765149.994736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020dace00 x1876702067954688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994758:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067954688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994770:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067954688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765149.994776:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765149.994794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994834:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020dae300 x1876702067954816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994845:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067954816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994853:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994856:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765149.994864:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765149.994883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765149.994893:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765149/real 1789765149] req@ffff936020dad180 x1876702067954944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765149.994903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067954944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765149.994909:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765149.994912:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765149.994915:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067954944:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765150.763077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765151.786932:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765152.810021:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765153.834176:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765154.859357:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765155.114129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.114139:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765155.114143:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.114152:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.114162:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.114169:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765155.114202:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067955072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.114230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.114234:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765155.114235:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.114239:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.114242:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.114257:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.114266:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765155.114276:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067955200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.114285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.114288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765155.114290:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.114293:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.114296:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789765155.114542:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067955072:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765155.114608:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.114643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef4700 x1876702067955072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.114660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067955072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.114676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.114679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.114683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067955072:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765155.114690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765155.114708:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.114719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef5180 x1876702067955200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.114729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067955200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.114737:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.114741:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.114744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955200:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765155.114936:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.114943:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765155.114955:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067955328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.114966:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.114970:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765155.114972:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.114986:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.114990:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.114994:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.114998:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765155.115006:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067955456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.115014:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.115017:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765155.115019:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.115022:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.115025:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.115028:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.115031:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765155.115039:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067955584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.115047:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.115050:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765155.115052:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.115055:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.115057:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.115060:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.115064:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765155.115070:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067955712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765155.115078:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765155.115081:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765155.115082:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765155.115085:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2622 00000100:00080000:1.0:1789765155.115088:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765155.115090:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765155.115094:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765155.115101:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067955840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765155.116094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765155.116134:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.116152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef4a80 x1876702067955328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.116166:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067955328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.116176:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.116179:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.116182:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765155.116189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765155.116205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.116214:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef5880 x1876702067955456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.116224:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067955456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.116232:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.116235:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.116238:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765155.116243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765155.116270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.116280:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef5180 x1876702067955584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.116289:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067955584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.116297:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.116313:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.116316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067955584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765155.116322:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067955712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765155.116339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.116349:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef4700 x1876702067955712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.116360:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067955712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.116367:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.116370:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.116373:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067955712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765155.116378:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067955840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765155.116392:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765155.116401:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765155/real 1789765155] req@ffff936010ef5c00 x1876702067955840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765155.116412:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067955840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765155.116419:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765155.116421:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765155.116424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067955840:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765155.882819:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765156.916272:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765157.930075:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765159.109964:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765159.978372:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765160.234534:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.234617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765160.234622:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.234635:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.234650:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.234661:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765160.234706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067955968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765160.234824:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067955968:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765160.234911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765160.235314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.235318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765160.235321:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.235326:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.235333:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.235337:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.235349:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765160.235366:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067956096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765160.236961:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa4000 x1876702067955968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.236996:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067955968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.237015:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.237021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.237026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067955968:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765160.237063:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956096:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765160.237328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.237334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765160.237336:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.237343:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.237351:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.237357:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.237370:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765160.237398:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067956224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765160.237412:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.237416:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765160.237418:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.237421:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.237426:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.237430:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.237435:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765160.237447:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067956352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765160.237462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.237464:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765160.237466:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.237479:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.237483:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.237487:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.237492:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765160.237503:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067956480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765160.237510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.237513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765160.237515:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.237518:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.237521:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.237524:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.237528:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765160.237537:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067956608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765160.237546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765160.237549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765160.237550:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765160.237553:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2628 00000100:00080000:3.0:1789765160.237557:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765160.237560:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765160.237565:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765160.237574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067956736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789765160.239410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239449:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa6300 x1876702067956096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239473:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067956096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239485:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239490:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765160.239527:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765160.239550:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239560:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa7b80 x1876702067956224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067956224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239581:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239585:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765160.239596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765160.239615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa4000 x1876702067956352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067956352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765160.239657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765160.239676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa7800 x1876702067956480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067956480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765160.239719:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067956608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765160.239738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa5c00 x1876702067956608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239760:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067956608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239767:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239771:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239774:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067956608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765160.239780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067956736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765160.239814:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765160.239826:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765160/real 1789765160] req@ffff936010aa5880 x1876702067956736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765160.239838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067956736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765185 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765160.239846:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765160.239850:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765160.239853:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067956736:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765161.002247:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765162.027530:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789765163.051442:0:255:0:(ldlm_pool.c:1330:ldlm_pools_recalc_task()) Negative interval(0) 00010000:00010000:0.0:1789765163.051622:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765164.079607:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765165.099681:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765165.354076:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354089:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765165.354094:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354107:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354121:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354130:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765165.354180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067956864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354211:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354215:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765165.354216:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354220:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354224:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354237:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354248:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765165.354261:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067956992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354270:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354274:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765165.354275:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354280:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354283:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354287:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354292:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765165.354302:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067957120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354311:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354314:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765165.354315:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354318:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354321:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354325:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354329:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765165.354338:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067957248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354346:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354349:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765165.354351:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354354:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354358:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354361:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354365:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765165.354374:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067957376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354383:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354387:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765165.354388:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354392:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354395:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354398:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354402:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765165.354411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067957504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765165.354420:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765165.354423:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765165.354425:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765165.354429:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2633 00000100:00080000:0.0:1789765165.354432:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765165.354436:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765165.354439:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765165.354448:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067957632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765165.354534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067956864:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765165.354631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.354698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020dac380 x1876702067956864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355134:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067956864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355153:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067956864:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765165.355170:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765165.355203:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355218:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020daf100 x1876702067956992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355241:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067956992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355248:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355255:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067956992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765165.355260:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765165.355281:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355292:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020dac700 x1876702067957120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355303:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067957120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355310:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355313:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765165.355325:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765165.355345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355355:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020daed80 x1876702067957248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067957248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355373:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355376:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355379:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765165.355384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765165.355403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355414:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020dae680 x1876702067957376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067957376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355433:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355436:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765165.355445:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067957504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765165.355466:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020dac000 x1876702067957504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355487:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067957504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355498:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355501:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067957504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765165.355506:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067957632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765165.355526:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765165.355537:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765165/real 1789765165] req@ffff936020dad880 x1876702067957632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765165.355549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067957632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765190 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765165.355556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765165.355559:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765165.355563:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067957632:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765166.122040:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765167.147398:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765168.171325:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765169.194078:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765170.218713:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765170.474857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.474868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765170.474872:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.474881:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.474891:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.474898:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765170.474935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067957760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.474968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.474972:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765170.474974:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.474977:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.474981:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.474985:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.474993:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765170.475005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067957888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.475013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.475016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765170.475017:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.475021:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.475024:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.475027:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.475031:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765170.475039:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067958016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.475047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.475050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765170.475051:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.475055:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.475059:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.475062:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.475066:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765170.475076:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067958144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.475084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.475086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765170.475087:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.475090:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.475093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.475096:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.475100:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765170.475108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067958272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.475114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.475117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765170.475119:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.475121:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.475123:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.475126:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.475129:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765170.475136:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067958400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765170.475148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765170.475151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765170.475152:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765170.475156:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2638 00000100:00080000:1.0:1789765170.475159:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765170.475162:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765170.475165:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765170.475172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067958528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765170.475667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067957760:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765170.475729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.475783:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef4000 x1876702067957760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.475832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067957760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.475847:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.475851:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.475854:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067957760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765170.475861:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765170.475884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.475896:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef4380 x1876702067957888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.475908:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067957888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.475915:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.475918:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.475921:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067957888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765170.475926:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765170.475945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.475955:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef7100 x1876702067958016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.475966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067958016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.475974:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.475976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.475980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765170.475986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765170.476006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.476017:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef4e00 x1876702067958144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.476029:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067958144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.476037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.476040:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.476043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765170.476048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765170.476067:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.476078:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef6d80 x1876702067958272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.476088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067958272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.476096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.476099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.476102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765170.476107:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067958400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765170.476127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.476138:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef6300 x1876702067958400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.476149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067958400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.476156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.476159:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.476162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067958400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765170.476168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067958528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765170.476187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765170.476211:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765170/real 1789765170] req@ffff936010ef7b80 x1876702067958528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765170.476223:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067958528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765170.476230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765170.476233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765170.476236:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067958528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765171.243597:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765172.267682:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765173.290280:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765174.318537:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765175.345221:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765175.594482:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594495:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765175.594500:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594513:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594527:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.594536:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765175.594588:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067958656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.594622:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594626:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765175.594628:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594632:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594636:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.594641:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.594653:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765175.594667:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067958784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.594678:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594681:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765175.594683:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594687:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594691:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.594695:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.594700:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765175.594712:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067958912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.594722:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594725:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765175.594727:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594731:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594734:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.594738:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.594743:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765175.594754:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067959040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.594763:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594766:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765175.594768:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594771:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594774:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.594778:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.594782:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765175.594791:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067959168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.594827:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.594831:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765175.594833:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.594836:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.594839:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789765175.595348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067958656:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765175.595421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.595465:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020dad500 x1876702067958656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.595483:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067958656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.595501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.595505:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.595509:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067958656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765175.595517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765175.595541:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.595553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020daf480 x1876702067958784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.595564:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067958784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.595571:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.595574:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.595577:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765175.595584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765175.595604:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.595615:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020dafb80 x1876702067958912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.595626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067958912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.595633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.595636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.595639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067958912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765175.595646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765175.595667:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.595679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020dad880 x1876702067959040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.595689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067959040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.595697:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.595701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.595703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765175.595709:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765175.595729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.595740:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020dac000 x1876702067959168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.595751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067959168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.595758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.595761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.595764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959168:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765175.595927:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.596065:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765175.596095:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067959296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765175.596123:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765175.596127:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765175.596130:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765175.596135:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2643 00000100:00080000:0.0:1789765175.596142:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765175.596148:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765175.597059:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765175.597101:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067959424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765175.598051:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067959296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765175.598126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.598178:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020dae680 x1876702067959296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.598197:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067959296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.598207:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.598212:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.598215:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067959296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765175.598223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067959424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765175.598245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765175.598256:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765175/real 1789765175] req@ffff936020daed80 x1876702067959424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765175.598267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067959424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765175.598274:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765175.598278:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765175.598281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067959424:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765176.363026:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765177.388614:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765178.410876:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765179.434910:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765180.458263:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765180.713979:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.713990:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765180.713994:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714005:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714015:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714022:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765180.714060:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067959552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714086:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714089:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765180.714091:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714095:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714098:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714101:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714109:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765180.714127:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067959680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714136:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714139:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765180.714142:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714145:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714148:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714151:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714155:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765180.714163:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067959808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714172:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714175:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765180.714176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714180:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714183:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714186:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714198:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765180.714206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067959936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714217:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765180.714218:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714221:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714224:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714233:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714237:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765180.714245:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067960064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765180.714258:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714261:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714263:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714266:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714269:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765180.714276:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067960192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765180.714284:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765180.714287:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765180.714289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765180.714292:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2648 00000100:00080000:0.0:1789765180.714294:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765180.714297:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765180.714301:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765180.714307:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067960320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765180.714876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067959552:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765180.714935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.714974:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020dac700 x1876702067959552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.714991:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067959552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715004:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715012:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067959552:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765180.715018:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765180.715037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020daf100 x1876702067959680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715060:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067959680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715066:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715069:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715072:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765180.715077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765180.715091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715112:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020dac380 x1876702067959808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715132:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067959808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715142:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765180.715150:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765180.715167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715178:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020dad180 x1876702067959936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715189:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067959936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715195:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067959936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765180.715206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765180.715221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715231:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020dae300 x1876702067960064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715240:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067960064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715246:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715249:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715251:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765180.715256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067960192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765180.715270:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715279:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020dace00 x1876702067960192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715288:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067960192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715294:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715297:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715300:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067960192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765180.715305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067960320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765180.715318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765180.715326:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765180/real 1789765180] req@ffff936020daea00 x1876702067960320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765180.715335:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067960320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765180.715341:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765180.715343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765180.715346:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067960320:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765181.482212:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765182.506067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765183.530604:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765184.558322:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765185.579220:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765185.834303:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765185.834317:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834336:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834346:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834361:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765185.834397:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067960448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765185.834426:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834430:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834433:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834439:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834448:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765185.834462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067960448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765185.834463:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067960576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765185.834483:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834487:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834491:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834494:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834499:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765185.834510:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067960704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834519:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765185.834523:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834526:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834529:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834533:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834537:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765185.834546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067960832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765185.834559:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834563:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834566:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834570:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834574:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765185.834583:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067960960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834597:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1789765185.834598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765185.834599:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834608:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834612:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765185.834625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067961088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765185.834633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765185.834636:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765185.834638:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765185.834641:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2653 00000100:00080000:3.0:1789765185.834645:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765185.834648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765185.834652:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765185.834659:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa4a80 x1876702067960448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765185.834665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067961216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765185.834693:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067960448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.834711:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.834715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.834719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067960448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765185.834740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765185.834761:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.834773:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa4380 x1876702067960576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.834784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067960576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.834792:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.834796:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.834825:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765185.834831:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765185.834849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.834859:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa4e00 x1876702067960704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.834870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067960704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.834878:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.834881:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.834884:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765185.834889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765185.834905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.834915:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa5f80 x1876702067960832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.834925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067960832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.834932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.834935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.834938:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765185.834943:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765185.834958:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.834968:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa7480 x1876702067960960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.834977:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067960960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.834985:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.834987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.834990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067960960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765185.834996:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765185.835325:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.835337:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa6680 x1876702067961088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.835350:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067961088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.835359:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.835362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.835365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765185.835371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765185.835390:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765185.835401:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765185/real 1789765185] req@ffff936010aa5180 x1876702067961216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765185.835413:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067961216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765185.835421:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765185.835424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765185.835428:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961216:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765186.602476:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765187.626575:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765188.651825:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765189.674539:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765190.710599:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765190.953958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.953967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765190.953970:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.953981:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.953991:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.953997:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765190.954035:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067961344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765190.954078:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954085:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954089:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954097:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765190.954108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067961472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765190.954122:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954128:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954134:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765190.954141:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067961600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954152:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765190.954153:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954156:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954158:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954161:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954164:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765190.954171:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067961728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765190.954183:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954186:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954188:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954191:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954194:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765190.954203:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067961856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954214:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765190.954216:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954219:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954221:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954224:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954228:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765190.954235:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067961984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765190.954242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765190.954252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765190.954254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765190.954257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2658 00000100:00080000:3.0:1789765190.954259:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765190.954262:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765190.954265:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765190.954273:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067962112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765190.954860:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067961344:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765190.954915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.954951:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa7100 x1876702067961344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.954967:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067961344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.954982:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.954986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.954990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067961344:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765190.954996:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765190.955016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa5500 x1876702067961472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955048:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067961472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765190.955069:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765190.955086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955096:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa6a00 x1876702067961600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067961600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955117:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955120:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765190.955125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765190.955140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955149:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa4700 x1876702067961728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955159:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067961728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955166:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955173:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765190.955177:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765190.955191:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955199:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa5180 x1876702067961856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955209:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067961856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955217:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067961856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765190.955227:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765190.955242:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa6680 x1876702067961984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955272:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067961984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955280:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955285:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067961984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765190.955291:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067962112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765190.955306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765190.955314:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765190/real 1789765190] req@ffff936010aa7480 x1876702067962112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765190.955323:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067962112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765190.955330:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765190.955332:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765190.955335:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067962112:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765191.724029:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765192.746647:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765193.769989:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765194.798568:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765195.818031:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765196.074185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765196.074198:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074208:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074219:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074227:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765196.074259:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067962240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765196.074293:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074296:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074300:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074303:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074312:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765196.074323:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067962368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074331:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074334:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765196.074335:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074338:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074341:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074344:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074348:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765196.074357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067962496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074426:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765196.074432:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074440:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074445:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074448:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074462:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765196.074479:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067962624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765196.074493:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074496:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074500:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074504:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074508:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765196.074518:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067962752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765196.074531:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074534:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074538:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074541:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074545:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765196.074553:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067962880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765196.074561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765196.074564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765196.074565:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765196.074568:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2663 00000100:00080000:3.0:1789765196.074571:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765196.074574:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765196.074577:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765196.074585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067963008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765196.074737:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067962240:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765196.074858:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765196.074889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765196.074909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765196.074928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765196.074947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067962880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765196.074966:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765196.842132:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765197.867218:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765198.892451:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765199.147702:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765199.147744:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765199.147760:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147830:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147841:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147848:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147856:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147864:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765199.147870:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765199.148038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa5f80 x1876702067962240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067962240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148073:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148077:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067962240:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765199.148089:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa4e00 x1876702067962368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148100:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067962368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148111:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148114:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765199.148120:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa4380 x1876702067962496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067962496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148156:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765199.148165:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa4a80 x1876702067962624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148175:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067962624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765199.148203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa5880 x1876702067962752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148213:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067962752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148226:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148230:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148233:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067962752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765199.148238:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa5c00 x1876702067962880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148268:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067962880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148276:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148279:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148282:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067962880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765199.148293:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765196/real 1789765199] req@ffff936010aa7800 x1876702067963008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765199.148304:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067963008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765199.148311:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765199.148314:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765199.148317:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765199.913961:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765200.939401:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765201.193891:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194445:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765201.194449:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194459:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194470:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194477:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765201.194524:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067963136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194549:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194552:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765201.194554:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194557:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194561:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194564:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194572:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765201.194584:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067963264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194593:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194596:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765201.194598:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194602:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194604:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194607:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194611:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765201.194619:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067963392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194627:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194630:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765201.194631:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194634:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194637:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194641:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194646:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765201.194656:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067963520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194665:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194668:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765201.194669:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194672:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194675:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194679:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194683:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765201.194692:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067963648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194700:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194703:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765201.194705:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194708:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194711:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194714:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194719:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765201.194728:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067963776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765201.194735:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765201.194738:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765201.194740:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765201.194743:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2669 00000100:00080000:3.0:1789765201.194746:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765201.194749:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765201.194753:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765201.194762:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067963904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765201.196058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067963136:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765201.196117:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.196154:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa4000 x1876702067963136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.196177:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067963136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.196188:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.196191:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.196195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067963136:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765201.196202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765201.196223:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.196235:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa7b80 x1876702067963264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.196244:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067963264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.196252:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.196255:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.196258:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765201.196264:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765201.199908:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.199944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa6300 x1876702067963392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.205653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067963392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.205668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.205672:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.205677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765201.205690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765201.205755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.205779:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa7800 x1876702067963520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.205822:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067963520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.205831:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.205834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.205836:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765201.205843:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765201.205864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.205876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa5c00 x1876702067963648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.205887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067963648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.205894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.205897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.205900:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067963648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765201.205906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765201.205926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.205937:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa5880 x1876702067963776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.205947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067963776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.208714:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.208728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.208734:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765201.208747:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765201.208837:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765201.208862:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765201/real 1789765201] req@ffff936010aa4a80 x1876702067963904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765201.208878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067963904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765201.208887:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765201.208891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765201.208895:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067963904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765201.963060:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765202.987986:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765204.010091:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765205.034013:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765206.062088:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765206.313975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.313995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765206.313998:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314009:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314020:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314028:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765206.314065:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067964032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765206.314102:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314106:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314110:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314113:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765206.314246:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067964032:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765206.314308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.314363:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa4380 x1876702067964032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.314382:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067964032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.314393:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.314396:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.314400:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067964032:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765206.314562:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067964160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765206.314587:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314592:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314600:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314606:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765206.314618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067964288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765206.314631:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314635:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314638:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314642:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765206.314656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067964416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765206.314667:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314671:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314673:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314677:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314681:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765206.314690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067964544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765206.314702:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314706:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314709:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314712:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765206.314727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067964672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765206.314736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765206.314739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765206.314740:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765206.314744:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2674 00000100:00080000:3.0:1789765206.314747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765206.314750:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765206.314753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765206.314762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067964800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765206.314829:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765206.314855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.314869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa4e00 x1876702067964160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.314882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067964160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.314890:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.314893:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.314896:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765206.314902:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765206.314922:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.314944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa5f80 x1876702067964288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.314955:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067964288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.314963:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.314966:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.314969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765206.314974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765206.314995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.315006:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa7480 x1876702067964416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.315017:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067964416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.315024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.315026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.315029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765206.315034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765206.315050:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.315058:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa6680 x1876702067964544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.315068:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067964544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.315075:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.315078:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.315081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067964544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765206.315085:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067964672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765206.315103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.315112:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa5180 x1876702067964672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.315123:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067964672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.315131:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.315596:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.315600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067964672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765206.315607:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067964800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765206.315632:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765206.315645:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765206/real 1789765206] req@ffff936010aa4700 x1876702067964800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765206.315657:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067964800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765231 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765206.315664:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765206.315667:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765206.315670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067964800:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765207.103991:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765208.108591:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765209.130872:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765210.155324:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765211.178151:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765211.433934:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.433943:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765211.433947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.433957:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.433966:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.433972:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765211.434006:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067964928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434032:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434036:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765211.434037:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434040:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434044:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434047:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434054:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765211.434064:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067965056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434073:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434076:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765211.434078:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434081:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434085:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434087:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434091:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765211.434099:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067965184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434107:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434110:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765211.434111:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434115:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434117:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434120:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434124:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765211.434133:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067965312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434141:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434143:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765211.434145:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434148:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434150:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434153:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765211.434174:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067965440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434188:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434191:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765211.434193:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434196:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434199:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434202:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434206:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765211.434214:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067965568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765211.434222:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765211.434225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765211.434227:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765211.434230:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2679 00000100:00080000:3.0:1789765211.434233:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765211.434236:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765211.434240:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765211.434248:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067965696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765211.434730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067964928:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765211.434788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.434858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa6a00 x1876702067964928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.434888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067964928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.434906:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.434910:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.434913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067964928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765211.434920:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765211.434944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.434957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa5500 x1876702067965056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.434968:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067965056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.434975:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.434978:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.434981:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765211.434987:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765211.435005:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.435016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa7100 x1876702067965184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.435026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067965184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.435033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.435036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.435039:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765211.435044:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765211.435063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.435073:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa4700 x1876702067965312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.435083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067965312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.435090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.435093:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.435096:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765211.435102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765211.435119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.435129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa5180 x1876702067965440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.435139:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067965440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.435147:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.435150:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.435153:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765211.435168:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067965568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765211.435188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.435198:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa6680 x1876702067965568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.435208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067965568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.435215:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.435218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.435221:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067965568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765211.435228:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067965696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765211.435246:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765211.435256:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765211/real 1789765211] req@ffff936010aa7480 x1876702067965696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765211.435267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067965696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765211.435273:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765211.435275:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765211.435278:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067965696:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765212.206998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765213.226017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765214.252081:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765215.274301:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765216.298608:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765216.555273:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555284:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765216.555289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555299:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555308:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555315:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765216.555349:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067965824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555374:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555377:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765216.555379:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555383:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555386:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555390:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555399:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765216.555411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067965952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555425:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555428:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765216.555429:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555434:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555436:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555439:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555443:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765216.555453:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067966080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555461:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555463:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765216.555465:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555468:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555471:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555475:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555479:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765216.555488:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067966208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555496:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555499:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765216.555500:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555503:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555506:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555509:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555513:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765216.555522:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067966336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555529:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555533:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765216.555534:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555537:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555540:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555543:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555548:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765216.555559:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067966464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765216.555568:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765216.555571:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765216.555573:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765216.555576:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2684 00000100:00080000:3.0:1789765216.555579:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765216.555582:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765216.555586:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765216.555594:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067966592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765216.555836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067965824:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765216.555902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.555956:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa5f80 x1876702067965824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.555971:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067965824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.555993:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.555996:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.555999:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067965824:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765216.556006:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765216.556026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa4e00 x1876702067965952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067965952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556054:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067965952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765216.556213:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765216.556234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556246:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa4380 x1876702067966080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556257:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067966080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556267:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556270:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765216.556276:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765216.556296:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556306:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa4a80 x1876702067966208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556318:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067966208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556325:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556328:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556330:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765216.556335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765216.556353:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa5880 x1876702067966336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556382:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067966336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556388:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556391:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556394:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765216.556399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067966464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765216.556416:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556426:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa5c00 x1876702067966464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067966464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556443:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556446:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556448:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067966464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765216.556453:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067966592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765216.556471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765216.556481:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765216/real 1789765216] req@ffff936010aa7800 x1876702067966592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765216.556491:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067966592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765216.556498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765216.556501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765216.556504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067966592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765217.325116:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765218.346090:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765219.370083:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765220.394174:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765221.418482:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765221.673950:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.673959:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765221.673963:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.673972:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.673982:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.673989:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765221.674023:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067966720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765221.674178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067966720:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765221.674235:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765221.674262:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.674265:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765221.674267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.674271:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.674276:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.674280:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.674288:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765221.674300:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067966848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765221.674428:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa6300 x1876702067966720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.674446:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067966720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.674456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.674460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.674464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067966720:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765221.674484:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765221.674505:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.674515:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa7b80 x1876702067966848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.674527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067966848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.674535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.674538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.674541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765221.674643:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.674650:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765221.674652:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.674657:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.674661:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.674665:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.674670:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765221.674682:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067966976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765221.674692:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.674695:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765221.674697:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.674700:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.674703:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.674706:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.674710:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765221.674718:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067967104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765221.674732:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.674735:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765221.674736:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.674739:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.674742:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.674745:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.674748:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765221.674755:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067967232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765221.674764:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789765221.674906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765221.674928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.674940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa4000 x1876702067966976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.674952:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067966976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.674961:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.674963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.674967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067966976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765221.674972:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765221.674990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.675000:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa7b80 x1876702067967104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.675012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067967104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.675020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.675023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.675026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765221.675031:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765221.675048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.675057:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa6300 x1876702067967232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.675068:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067967232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.675076:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.675079:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.675082:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967232:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765221.675137:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765221.675139:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.675143:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.675147:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.675151:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.675156:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765221.675167:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067967360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765221.675184:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765221.675188:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765221.675189:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765221.675193:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2689 00000100:00080000:3.0:1789765221.675196:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765221.675200:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765221.675204:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765221.675212:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067967488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765221.675264:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067967360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765221.675284:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.675295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa7800 x1876702067967360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.675307:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067967360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.675315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.675318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.675321:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067967360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765221.675327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067967488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765221.675341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765221.675350:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765221/real 1789765221] req@ffff936010aa5c00 x1876702067967488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765221.675360:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067967488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765221.675368:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765221.675370:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765221.675374:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067967488:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765222.442155:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765223.466017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765224.490378:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765225.514450:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765226.538122:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765226.794022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.794031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765226.794035:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.794046:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.794055:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.794062:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765226.794096:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067967616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.794180:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067967616:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765226.794251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765226.794281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.794285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765226.794287:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.794292:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.794296:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.794300:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.794309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765226.794323:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067967744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.796986:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa5880 x1876702067967616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.797014:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067967616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.797025:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.797029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.797033:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067967616:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765226.797064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765226.797113:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765226.797128:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa4a80 x1876702067967744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.797139:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067967744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.797147:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.797151:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.797155:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967744:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765226.800028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.800037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765226.800041:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.800050:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.800060:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.800067:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.800078:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765226.800111:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067967872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.800162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765226.800226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765226.800254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.800258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765226.800260:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.800264:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.800268:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.800272:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.800277:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765226.800289:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067968000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.800401:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa4380 x1876702067967872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.800420:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067967872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.800430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.800433:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.800437:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067967872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765226.800458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765226.800481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765226.800493:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa4e00 x1876702067968000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.800504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067968000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.800511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.800514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.800517:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968000:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765226.800755:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.800760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765226.800761:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.800766:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.800862:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.800866:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.800872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765226.800884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067968128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.800928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765226.800952:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765226.800973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.800976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765226.800978:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.800982:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.800986:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.800989:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.800994:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765226.801004:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067968256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.801040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa5f80 x1876702067968128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.801051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067968128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.801058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.801061:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.801064:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765226.801078:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067968256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765226.801098:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765226.801109:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa7480 x1876702067968256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.801120:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067968256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.801128:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.801131:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.801134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067968256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765226.801275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765226.801280:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765226.801282:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765226.801286:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2694 00000100:00080000:3.0:1789765226.801291:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765226.801294:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765226.801299:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765226.801311:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067968384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765226.801351:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067968384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765226.801373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765226.801436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765226/real 1789765226] req@ffff936010aa6680 x1876702067968384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765226.801448:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067968384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765226.801455:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765226.801458:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765226.801461:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067968384:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765227.562022:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765228.586071:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765229.610036:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765230.634186:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765231.658067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765231.913863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.913873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765231.913878:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.913888:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.913899:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.913906:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765231.913941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067968512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765231.914044:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914048:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914053:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914058:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914067:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765231.914084:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067968640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765231.914100:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914104:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914112:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914117:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765231.914128:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067968768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765231.914148:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914152:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914155:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914158:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914162:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765231.914172:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067968896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765231.914184:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914188:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914191:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914194:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914198:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765231.914206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067969024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765231.914219:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914222:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914225:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914229:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765231.914241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067969152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765231.914250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765231.914252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765231.914254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765231.914257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2699 00000100:00080000:3.0:1789765231.914260:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765231.914263:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765231.914267:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765231.914276:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067969280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765231.914486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067968512:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765231.914547:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.914582:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa5180 x1876702067968512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.914600:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067968512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.914611:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.914614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.914618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067968512:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765231.914625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765231.914649:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.914662:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa4700 x1876702067968640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.914673:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067968640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.914681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.914684:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.914700:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765231.914707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765231.914727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.914752:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa7100 x1876702067968768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.914765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067968768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.914773:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.914776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.914778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765231.914784:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765231.914834:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.914845:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa5500 x1876702067968896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.914855:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067968896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.914863:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.914866:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.914869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067968896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765231.914876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765231.914898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.914909:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa6a00 x1876702067969024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.914920:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067969024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.914927:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.914930:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.914933:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765231.914939:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067969152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765231.914957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.915094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa6680 x1876702067969152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.915108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067969152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.915116:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.915119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.915122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067969152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765231.915128:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067969280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765231.915148:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765231.915158:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765231/real 1789765231] req@ffff936010aa7480 x1876702067969280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765231.915169:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067969280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765231.915176:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765231.915179:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765231.915182:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067969280:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765232.682700:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765233.705926:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765234.730126:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765235.754980:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765236.778341:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765237.034053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.034061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765237.034065:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.034074:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.034083:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.034089:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765237.034875:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067969408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.034909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.034913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765237.034915:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.034919:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.034925:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.034929:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.034937:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765237.034949:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067969536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.034958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.034961:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765237.034962:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.034966:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.034969:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.034973:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.034986:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765237.034997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067969664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.035005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.035008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765237.035009:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.035012:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.035016:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.035019:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.035024:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765237.035033:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067969792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.035040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.035043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765237.035045:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.035048:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.035051:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.035054:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.035058:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765237.035066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067969920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.035073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.035076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765237.035078:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.035080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.035083:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.035086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.035090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765237.035402:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067969408:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765237.035455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.035495:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa5f80 x1876702067969408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.035511:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067969408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.035521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.035525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.035529:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067969408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765237.035538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765237.035558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.035569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa4e00 x1876702067969536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.035580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067969536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.035588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.035591:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.035594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765237.035599:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765237.035613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.035621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa4380 x1876702067969664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.035631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067969664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.035638:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.035641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.035644:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765237.035648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765237.035663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.035671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa4a80 x1876702067969792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.035682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067969792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.035689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.035691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.035694:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765237.035699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765237.035726:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.035737:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa5880 x1876702067969920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.035749:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067969920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.035756:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.035758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.035761:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067969920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765237.035912:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067970048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765237.035932:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765237.035935:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765237.035937:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765237.035941:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2704 00000100:00080000:3.0:1789765237.035946:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765237.035951:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765237.035956:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765237.035969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067970176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765237.036269:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765237.036292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.036303:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa5c00 x1876702067970048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.036314:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067970048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.036328:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.036331:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.036334:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765237.036339:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765237.036353:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765237.036361:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765237/real 1789765237] req@ffff936010aa7800 x1876702067970176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765237.036371:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067970176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765237.036378:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765237.036381:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765237.036383:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765237.801967:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765238.826674:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765239.850084:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765240.874477:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765241.898212:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765242.154424:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.154434:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765242.154439:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.154448:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.154459:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.154467:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765242.154502:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067970304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.154593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067970304:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765242.154652:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765242.154693:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.154697:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765242.154699:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.154703:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.154708:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.154712:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.154723:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765242.154735:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067970432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.155168:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef7480 x1876702067970304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.155199:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067970304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.155214:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.155218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.155222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067970304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765242.155641:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765242.155683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765242.155698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef6680 x1876702067970432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.155711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067970432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.155720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.155723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.155726:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970432:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765242.155982:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.155988:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765242.155990:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.155995:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.156000:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.156005:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.156013:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765242.156029:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067970560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.156076:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765242.156101:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765242.156122:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.156126:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765242.156128:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.156132:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.156136:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.156140:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.156145:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765242.156156:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067970688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.156398:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef7800 x1876702067970560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.156412:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067970560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.156421:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.156424:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.156428:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765242.156441:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765242.156464:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765242.156477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef5500 x1876702067970688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.156489:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067970688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.156497:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.156500:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.156503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970688:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789765242.156682:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.156686:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765242.156689:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.156693:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.156697:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.156701:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.156707:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765242.156719:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067970816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.156859:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765242.156884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765242.156937:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.156941:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765242.156943:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.156948:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.156952:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.156957:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.156962:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765242.156975:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067970944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.157135:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef6a00 x1876702067970816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.157150:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067970816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.157159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.157163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.157166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067970816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765242.157311:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765242.157397:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765242.157410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef5f80 x1876702067970944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.157422:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067970944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.157430:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.157433:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.157436:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067970944:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765242.157526:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765242.157530:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765242.157532:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765242.157537:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2709 00000100:00080000:1.0:1789765242.157542:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765242.157546:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765242.157604:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765242.157624:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067971072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765242.157675:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765242.157698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765242.157950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765242/real 1789765242] req@ffff936010ef5f80 x1876702067971072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765242.157964:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067971072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765242.157973:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765242.157977:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765242.157980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971072:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765242.923210:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765243.946068:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765244.970148:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765245.994060:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765247.018020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765247.274641:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.274651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765247.274655:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.274665:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.274675:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.274682:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765247.274717:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067971200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765247.274845:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067971200:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765247.274903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765247.274949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.274953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765247.274955:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.274960:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.274964:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.274967:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.274976:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765247.274990:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067971328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765247.277482:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa6300 x1876702067971200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.277516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067971200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.277528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.277533:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.277537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067971200:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765247.277566:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765247.277617:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765247.277643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa7b80 x1876702067971328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.277656:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067971328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.277664:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.277668:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.277671:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971328:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765247.278557:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.278564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765247.278567:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.278573:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.278579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.278586:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.278596:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765247.278622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067971456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765247.278679:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765247.278713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765247.278738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.278751:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765247.278753:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.278757:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.278762:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.278765:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.278771:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765247.278784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067971584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765247.284690:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa4000 x1876702067971456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.284719:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067971456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.284736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.284752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.284756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765247.284790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765247.284866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765247.284883:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa7b80 x1876702067971584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.284894:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067971584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.284902:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.284905:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.284908:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971584:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765247.285073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.285078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765247.285081:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.285086:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.285093:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.285100:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.285109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765247.285134:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067971712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765247.285146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.285150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765247.285152:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.285155:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.285158:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.285162:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.285166:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765247.285178:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067971840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765247.285186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765247.285189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765247.285191:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765247.285194:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2715 00000100:00080000:3.0:1789765247.285197:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765247.285201:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765247.285204:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765247.285213:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067971968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765247.287061:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765247.287126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765247.287150:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa6300 x1876702067971712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.287167:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067971712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.287178:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.287183:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.287186:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067971712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765247.287194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765247.287215:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765247.287228:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa7800 x1876702067971840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.287239:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067971840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.287247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.287250:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.287253:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765247.287260:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765247.287279:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765247.287289:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765247/real 1789765247] req@ffff936010aa5c00 x1876702067971968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765247.287299:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067971968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765272 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765247.287306:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765247.287309:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765247.287312:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067971968:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765248.042492:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765249.065964:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765250.090496:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765251.115753:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765252.138151:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765252.394651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765252.394666:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394675:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394685:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394692:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765252.394729:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067972096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765252.394766:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394769:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394776:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394784:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765252.394795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067972224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765252.394854:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394858:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394861:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394864:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394868:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765252.394878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067972352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394885:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765252.394889:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394892:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394895:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394898:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394901:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765252.394909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067972480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394916:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765252.394920:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394924:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394927:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394930:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394934:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765252.394941:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067972608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765252.394951:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394954:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394957:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394960:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394963:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765252.394970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067972736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765252.394975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765252.394978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765252.394979:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765252.394982:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2720 00000100:00080000:1.0:1789765252.394984:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765252.394987:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765252.394990:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765252.394996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067972864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765252.395094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067972096:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765252.395169:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef6a00 x1876702067972096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395221:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067972096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395231:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395235:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395238:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067972096:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765252.395245:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765252.395265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395277:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef5500 x1876702067972224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395288:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067972224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395295:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395298:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395301:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765252.395307:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765252.395324:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395335:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef7800 x1876702067972352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395344:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067972352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395354:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765252.395361:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765252.395377:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef6680 x1876702067972480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395398:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067972480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765252.395417:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765252.395434:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395449:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef7480 x1876702067972608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067972608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395476:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395479:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067972608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765252.395483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067972736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765252.395506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef7b80 x1876702067972736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067972736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067972736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765252.395545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067972864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765252.395561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765252.395570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765252/real 1789765252] req@ffff936010ef6300 x1876702067972864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765252.395579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067972864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765277 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765252.395586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765252.395589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765252.395601:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067972864:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765253.162077:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765254.187153:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765255.210173:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765256.234641:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765257.258259:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765257.513923:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.513940:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765257.513944:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.513954:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.513965:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765257.513973:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765257.514008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067972992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765257.514038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.514042:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765257.514043:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.514233:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.514238:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765257.514242:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765257.514252:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765257.514266:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067973120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765257.514281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067972992:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765257.514290:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.514297:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765257.514299:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.514304:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.514309:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765257.514312:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765257.514317:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765257.514329:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067973248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765257.514338:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.514341:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765257.514343:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.514348:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2725 00000800:00000100:0.0:1789765257.514349:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765257.514353:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765257.514358:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765257.514363:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765257.514385:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067973376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765257.514394:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765257.514399:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef6d80 x1876702067972992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765257.514400:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765257.514403:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.514411:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.514418:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765257.514423:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514427:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067972992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789765257.514428:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765257.514443:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789765257.514446:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067973504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765257.514450:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514458:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067972992:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765257.514473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.514476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765257.514478:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765257.514483:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.514487:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765257.514491:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973120:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765257.514503:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765257.514509:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765257.514518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765257.514524:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067973632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765257.514534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef4e00 x1876702067973120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765257.514537:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765257.514546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765257.514566:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765257.514570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067973120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765257.514574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2725 00000100:00080000:1.0:1789765257.514582:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765257.514584:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765257.514589:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789765257.514592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789765257.514597:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765257.514598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765257.514610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765257.514615:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067973760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765257.514636:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514647:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef7100 x1876702067973248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765257.514659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067973248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765257.514666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765257.514669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765257.514677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765257.514693:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef4380 x1876702067973376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765257.514711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067973376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765257.514718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765257.514721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514723:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765257.514728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765257.514742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514750:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef4000 x1876702067973504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765257.514761:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067973504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765257.514767:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765257.514770:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514773:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067973504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765257.514786:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067973632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765257.514839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514849:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef5c00 x1876702067973632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765257.514859:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067973632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765257.514866:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765257.514869:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514872:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067973632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765257.514877:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067973760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765257.514892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765257.514901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765257/real 1789765257] req@ffff936010ef4700 x1876702067973760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765257.514911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067973760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765257.514924:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765257.514927:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765257.514930:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067973760:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765258.283004:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765259.306587:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765260.331694:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765261.355819:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765262.378019:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765262.635528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765262.635542:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635552:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635563:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635569:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765262.635602:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067973888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765262.635640:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635644:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635647:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635651:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635659:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765262.635671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067974016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765262.635685:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635689:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635700:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765262.635711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067974144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635722:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765262.635723:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635726:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635729:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635732:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765262.635744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067974272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765262.635757:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635760:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635763:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635766:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635769:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765262.635776:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067974400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635785:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765262.635787:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635789:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635795:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635817:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765262.635826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067974528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765262.635834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765262.635836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765262.635838:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765262.635841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2730 00000100:00080000:1.0:1789765262.635844:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765262.635847:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765262.635850:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765262.635858:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067974656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765262.636480:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067973888:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765262.636640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765262.636667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765262.636684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765262.636699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765262.636714:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067974528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765262.636731:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067974656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765263.402125:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765264.425992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765265.450146:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765265.706091:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765265.706124:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765265.706137:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706175:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706184:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706193:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706202:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706211:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765265.706221:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765265.706223:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef5180 x1876702067973888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706261:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067973888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067973888:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765265.706293:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef5880 x1876702067974016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706305:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067974016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706312:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706315:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706319:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765265.706325:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef4a80 x1876702067974144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706335:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067974144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706341:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706344:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765265.706363:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef4700 x1876702067974272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706373:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067974272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706379:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706382:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706385:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765265.706390:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef5c00 x1876702067974400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067974400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765265.706415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef4000 x1876702067974528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706423:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067974528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706429:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706432:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067974528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765265.706440:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765262/real 1789765265] req@ffff936010ef4380 x1876702067974656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765265.706449:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067974656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765265.706455:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765265.706457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765265.706460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067974656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765266.474585:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765267.497971:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765267.754063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.754076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765267.754079:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.754088:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.754099:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.754106:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765267.754139:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067974784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765267.755577:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755585:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755595:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755601:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755613:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765267.755638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067974912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765267.755653:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755657:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755660:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755663:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755667:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765267.755678:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067975040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765267.755691:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755694:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755697:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755700:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765267.755716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067975168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765267.755728:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755731:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755734:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755737:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755741:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765267.755749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067975296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755759:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765267.755760:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765267.755780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067975424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765267.755788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765267.755791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765267.755792:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765267.755811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2735 00000100:00080000:1.0:1789765267.755815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765267.755819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765267.755824:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765267.755833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067975552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765267.758390:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067974784:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765267.758495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.758540:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef7100 x1876702067974784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.758558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067974784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.758570:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.758574:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.758578:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067974784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765267.758586:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765267.758609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.758621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef4e00 x1876702067974912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.758633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067974912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.758642:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.758645:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.758649:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067974912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765267.758656:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765267.758676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.758697:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef6d80 x1876702067975040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.758714:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067975040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.758724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.758728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.758733:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765267.758741:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765267.758763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.758775:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef6300 x1876702067975168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.758790:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067975168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.758823:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.760357:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.760368:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765267.760379:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765267.760429:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.760531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef7b80 x1876702067975296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.760549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067975296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.760559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.760563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.760566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765267.760573:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067975424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765267.760595:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.760606:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef7480 x1876702067975424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.760617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067975424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.760625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.760628:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.760631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067975424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765267.760637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067975552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765267.760655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765267.760665:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765267/real 1789765267] req@ffff936010ef6680 x1876702067975552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765267.760676:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067975552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765267.761085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765267.761091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765267.761095:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067975552:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765268.522674:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765269.546073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765270.570510:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765271.594055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765272.618205:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765272.874016:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874030:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765272.874035:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874047:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874061:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874070:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765272.874134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067975680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874165:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874169:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765272.874171:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874176:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874179:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874183:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874193:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765272.874205:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067975808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874227:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765272.874229:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874234:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874238:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874241:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874247:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765272.874259:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067975936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874268:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874272:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765272.874273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874277:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874280:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874284:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874289:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765272.874298:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067976064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874306:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874309:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765272.874311:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874314:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874317:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874320:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874325:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765272.874334:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067976192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874342:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874345:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765272.874348:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874352:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874355:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874359:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874363:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765272.874373:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067976320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765272.874381:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765272.874385:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765272.874386:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765272.874390:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2740 00000100:00080000:3.0:1789765272.874393:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765272.874396:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765272.874400:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765272.874409:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067976448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765272.874526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067975680:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765272.874608:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.874668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa5880 x1876702067975680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.874686:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067975680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.874696:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.874700:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.874705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067975680:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765272.874712:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765272.874748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.874762:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa4a80 x1876702067975808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.874774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067975808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.874787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.874791:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.874794:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765272.874843:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765272.874868:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.874880:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa4380 x1876702067975936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.874893:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067975936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.874900:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.874903:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.874906:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067975936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765272.874912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765272.874934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.874944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa4e00 x1876702067976064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.874954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067976064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.874961:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.874963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.874967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765272.874973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765272.874991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.875002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa5f80 x1876702067976192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.875013:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067976192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.875020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.875023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.875025:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765272.875030:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067976320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765272.875049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.875059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa7480 x1876702067976320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.875071:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067976320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.875077:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.875080:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.875083:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067976320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765272.875088:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067976448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765272.875106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765272.875118:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765272/real 1789765272] req@ffff936010aa6680 x1876702067976448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765272.875129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067976448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765272.875137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765272.875140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765272.875144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067976448:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765273.642330:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765274.670695:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765275.689973:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765276.714094:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765277.737902:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765277.994509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765277.994519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765277.994523:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765277.994533:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765277.994543:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765277.994551:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765277.994589:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067976576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765277.994655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067976576:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765277.994714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765277.994741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765277.994745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765277.994747:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765277.994751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765277.994755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765277.994760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765277.994768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765277.994783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067976704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765277.994870:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765277/real 1789765277] req@ffff936010ef7800 x1876702067976576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765277.994888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067976576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765277.994898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765277.994901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765277.994905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067976576:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765277.994927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765277.994951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765277.994964:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765277/real 1789765277] req@ffff936010ef5500 x1876702067976704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765277.994976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067976704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765277.994983:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765277.994986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765277.994989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976704:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765277.998241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765277.998251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765277.998254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765277.998264:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765277.998274:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765277.998282:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765277.998293:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765277.998325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067976832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765277.998376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765277.998512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765277.998538:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765277.998542:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765277.998544:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765277.998547:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765277.998552:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765277.998556:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765277.998561:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765277.998574:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067976960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765278.000562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765277/real 1789765277] req@ffff936010ef6a00 x1876702067976832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765278.000585:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067976832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765278.000596:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765278.000600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765278.000604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765278.000629:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765278.000664:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765278.000676:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765277/real 1789765278] req@ffff936010ef5f80 x1876702067976960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765278.000687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067976960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765278.000701:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765278.000704:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765278.000707:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067976960:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789765278.002200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765278.002208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765278.002211:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765278.002221:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765278.002229:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765278.002236:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765278.002246:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765278.002275:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702067977088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765278.002289:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765278.002292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765278.002294:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765278.002297:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765278.002301:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765278.002304:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765278.002309:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765278.002318:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067977216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765278.002328:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765278.002331:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765278.002333:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765278.002336:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2745 00000100:00080000:1.0:1789765278.002339:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765278.002342:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765278.002346:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765278.002354:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067977344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765278.002505:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765278.002548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765278.002567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765278/real 1789765278] req@ffff936010ef5f80 x1876702067977088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765278.002582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702067977088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765278.002591:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765278.002595:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765278.002599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765278.002605:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067977216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765278.002623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765278.002633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765278/real 1789765278] req@ffff936010ef6a00 x1876702067977216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765278.002645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067977216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765278.002653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765278.002656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765278.002659:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067977216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765278.002664:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067977344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765278.002680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765278.002689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765278/real 1789765278] req@ffff936010ef5500 x1876702067977344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765278.002700:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067977344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765278.002707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765278.002710:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765278.002714:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067977344:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765278.762007:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765279.786117:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765280.810688:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765281.834132:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765282.859938:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765283.115159:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115168:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765283.115172:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115182:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115191:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115197:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765283.115232:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067977472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115267:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115271:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765283.115273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115277:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115281:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115285:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115294:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765283.115306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067977600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115315:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115318:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765283.115320:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115324:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115327:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115330:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115335:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765283.115346:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067977728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115354:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115356:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765283.115358:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115361:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115380:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115384:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115388:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765283.115398:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067977856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115406:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115408:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765283.115410:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115413:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115416:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115419:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115423:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765283.115432:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067977984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115440:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115443:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765283.115444:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115447:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115450:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115453:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115456:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765283.115465:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067978112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765283.115488:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765283.115492:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765283.115494:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765283.115497:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2750 00000100:00080000:3.0:1789765283.115500:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765283.115503:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765283.115507:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765283.115516:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067978240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765283.115647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067977472:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765283.115700:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.115745:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa6a00 x1876702067977472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.115763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067977472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.115771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.115775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.115778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067977472:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765283.115787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765283.115847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.115860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa5500 x1876702067977600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.115873:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067977600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.115883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.115886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.115889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765283.115896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765283.115915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.115925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa7100 x1876702067977728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.115936:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067977728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.115942:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.115945:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.115948:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765283.115953:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765283.115972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.115982:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa4700 x1876702067977856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.115994:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067977856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.116001:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.116003:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.116006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765283.116011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765283.116029:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.116040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa5180 x1876702067977984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.116051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067977984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.116058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.116062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.116064:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067977984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765283.116069:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067978112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765283.116087:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.116098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa6680 x1876702067978112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.116109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067978112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.116121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.116124:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.116127:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067978112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765283.116132:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067978240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765283.116151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765283.116161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765283/real 1789765283] req@ffff936010aa7480 x1876702067978240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765283.116171:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067978240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765283.116179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765283.116181:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765283.116184:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067978240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765283.882046:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765284.906672:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765285.930819:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765286.956969:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765287.979142:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765288.234030:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234040:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765288.234043:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234052:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234061:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234069:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765288.234103:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067978368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234136:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234140:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765288.234141:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234145:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234149:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234152:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234160:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765288.234171:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067978496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234179:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234182:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765288.234183:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234186:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234189:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234192:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234196:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765288.234204:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067978624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234212:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234214:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765288.234215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234218:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234221:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234224:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234228:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765288.234237:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067978752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234244:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234247:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765288.234248:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234251:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234254:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234257:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234261:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765288.234270:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067978880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234277:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234280:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765288.234282:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234284:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234287:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234291:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234295:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765288.234303:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067979008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765288.234317:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765288.234320:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765288.234321:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765288.234388:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2756 00000100:00080000:0.0:1789765288.234393:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765288.234397:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765288.234402:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765288.234414:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067979136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765288.234507:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067978368:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765288.234565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.234616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020dadf80 x1876702067978368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.234634:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067978368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.234644:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.234648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.234651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067978368:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765288.234658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765288.234679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.234691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020dadc00 x1876702067978496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.234706:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067978496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.234713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.234716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.234719:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765288.234724:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765288.234743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.234754:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020daf800 x1876702067978624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.234764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067978624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.234772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.234774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.234777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765288.234782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765288.234828:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.234917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020daea00 x1876702067978752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.234930:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067978752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.234941:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.234944:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.234947:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765288.234953:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765288.234974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.234987:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020dace00 x1876702067978880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.234997:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067978880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.235004:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.235008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.235011:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067978880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765288.235017:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765288.235035:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.235046:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020dae300 x1876702067979008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.235057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067979008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.235065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.235068:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.235071:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765288.235077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765288.235096:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765288.235107:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765288/real 1789765288] req@ffff936020dad180 x1876702067979136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765288.235118:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067979136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765313 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765288.235124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765288.235127:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765288.235130:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765289.002077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765290.026758:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765291.050044:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765292.075012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765293.098221:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765293.354027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.354038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765293.354042:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.354052:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.354062:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.354069:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765293.354107:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067979264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765293.354173:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067979264:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765293.354237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765293.354266:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.354271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765293.354273:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.354277:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.354281:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.354286:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.354303:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765293.354317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067979392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765293.355513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061dfb80 x1876702067979264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.355536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067979264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.355547:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.355553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.355557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067979264:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765293.355584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765293.355624:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.355638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061de680 x1876702067979392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.355651:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067979392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.355659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.355662:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.355665:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979392:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765293.355850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.355855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765293.355857:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.355861:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.355866:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.355871:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.355879:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765293.355901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702067979520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765293.355913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.355916:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765293.355918:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.355921:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.355925:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.355929:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.355933:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765293.355942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702067979648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765293.355950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.355953:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765293.355955:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.355957:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.355960:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.355963:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.355967:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765293.355975:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702067979776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765293.355982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.355985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765293.355986:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.355989:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.355991:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.355995:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.355998:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765293.356006:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702067979904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765293.356013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765293.356015:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765293.356016:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765293.356019:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2761 00000100:00080000:2.0:1789765293.356022:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765293.356025:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765293.356029:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765293.356036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702067980032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765293.357405:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765293.357459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.357482:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061de680 x1876702067979520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.357497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702067979520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.357507:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.357511:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.357514:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765293.357521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765293.357542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.357554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061dfb80 x1876702067979648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.357572:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702067979648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.357580:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.357583:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.357586:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765293.357592:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765293.357612:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.357623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061df800 x1876702067979776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.357633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702067979776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.357641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.357644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.357647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067979776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765293.357652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765293.357670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.357680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061dca80 x1876702067979904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.357690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702067979904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.357702:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.357705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.357709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067979904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765293.357714:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765293.357732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765293.357741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765293/real 1789765293] req@ffff9360061dd180 x1876702067980032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765293.357752:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702067980032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765318 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765293.357760:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765293.357763:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765293.357765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980032:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765294.121996:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765295.146291:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765296.170912:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765297.193963:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765298.218077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765298.474874:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.474883:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765298.474886:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.474896:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.474906:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.474913:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765298.474948:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067980160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.474980:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.474983:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765298.474985:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.474989:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.474993:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.474996:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475006:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765298.475017:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067980288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.475026:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.475029:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765298.475031:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.475035:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.475038:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.475041:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475046:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765298.475065:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067980416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.475073:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.475076:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765298.475077:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.475080:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.475083:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.475087:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475091:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765298.475100:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067980544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.475108:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.475111:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765298.475113:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.475116:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.475119:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.475122:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475127:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765298.475136:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067980672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.475144:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.475147:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765298.475149:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.475152:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.475155:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.475158:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475162:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765298.475170:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067980800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765298.475178:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765298.475181:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765298.475182:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765298.475186:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2766 00000100:00080000:3.0:1789765298.475188:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765298.475191:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765298.475195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765298.475203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067980928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765298.475393:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067980160:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765298.475449:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475500:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa5f80 x1876702067980160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067980160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475531:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067980160:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765298.475545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765298.475570:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475582:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa4e00 x1876702067980288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067980288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765298.475609:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765298.475626:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475635:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa4380 x1876702067980416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067980416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475651:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475654:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765298.475662:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765298.475680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475690:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa4a80 x1876702067980544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067980544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475705:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475708:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765298.475864:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765298.475888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa5880 x1876702067980672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067980672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475918:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475921:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067980672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765298.475931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765298.475950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.475960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa5c00 x1876702067980800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.475972:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067980800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.475979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.475982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.475985:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765298.475990:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765298.476009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765298.476019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765298/real 1789765298] req@ffff936010aa7800 x1876702067980928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765298.476031:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067980928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765298.476038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765298.476041:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765298.476043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067980928:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765299.241963:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765300.266910:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765301.292597:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765302.317568:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765303.338892:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765303.595046:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595053:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765303.595057:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595066:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595076:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595083:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765303.595115:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067981056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595145:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595149:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765303.595150:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595154:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595158:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595161:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595171:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765303.595182:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067981184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595194:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765303.595196:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595200:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595206:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765303.595230:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067981312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595238:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595241:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765303.595243:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595250:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595253:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595256:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595261:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765303.595269:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067981440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595285:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595288:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765303.595290:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595293:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595296:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595298:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595302:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765303.595311:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067981568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595320:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595323:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765303.595324:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595328:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595330:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595333:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595336:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765303.595343:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067981696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765303.595352:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765303.595355:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765303.595356:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765303.595366:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2771 00000100:00080000:0.0:1789765303.595369:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765303.595371:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765303.595375:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765303.595382:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067981824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765303.595558:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067981056:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765303.595610:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.595643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020dac380 x1876702067981056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.595659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067981056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.595669:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.595673:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.595677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067981056:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765303.595684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765303.595707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.595720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020daf100 x1876702067981184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.595732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067981184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.595739:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.595742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.595745:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765303.595750:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765303.595771:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.595783:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020dac700 x1876702067981312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.595795:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067981312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.595831:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.595835:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.595838:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765303.595844:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765303.595866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.595878:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020daed80 x1876702067981440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.595890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067981440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.595897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.595900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.595903:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765303.595908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765303.595928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.595940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020dae680 x1876702067981568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.595958:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067981568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.595965:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.595968:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.595971:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067981568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765303.595977:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067981696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765303.595995:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.596007:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020dac000 x1876702067981696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.596017:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067981696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.596024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.596026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.596029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067981696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765303.596034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067981824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765303.596052:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765303.596063:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765303/real 1789765303] req@ffff936020dad880 x1876702067981824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765303.596072:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067981824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765303.596083:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765303.596086:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765303.596088:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067981824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765304.362007:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765305.386877:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765306.410133:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765307.434117:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765308.457997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765308.713916:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.713926:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765308.713930:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.713939:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.713951:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.713969:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765308.714006:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067981952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714031:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714036:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765308.714038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714041:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714045:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714049:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714057:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765308.714068:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067982080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714078:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714081:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765308.714083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714087:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714090:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714094:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714098:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765308.714107:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067982208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714116:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714119:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765308.714120:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714123:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714127:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714130:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714135:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765308.714144:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067982336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714154:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714157:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765308.714158:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714162:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714165:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714168:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714171:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765308.714273:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067982464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714284:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714287:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765308.714289:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714293:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714296:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714300:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714318:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765308.714330:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067982592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765308.714340:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765308.714343:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765308.714345:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765308.714349:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2776 00000100:00080000:0.0:1789765308.714352:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765308.714356:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765308.714361:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765308.714370:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067982720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765308.715347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067981952:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765308.715411:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020dafb80 x1876702067981952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067981952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715496:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715499:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067981952:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765308.715509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765308.715531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715542:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020daf480 x1876702067982080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715552:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067982080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765308.715570:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765308.715587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020dad500 x1876702067982208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067982208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715615:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715618:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765308.715623:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765308.715641:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715652:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020dad880 x1876702067982336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715662:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067982336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715670:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715674:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715678:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765308.715690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765308.715709:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020dac000 x1876702067982464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067982464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715738:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715741:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765308.715750:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067982592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765308.715770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715782:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020dae680 x1876702067982592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715814:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067982592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715821:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715824:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715827:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067982592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765308.715833:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067982720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765308.715863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765308.715874:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765308/real 1789765308] req@ffff936020daed80 x1876702067982720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765308.715883:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067982720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765308.715895:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765308.715898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765308.715913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067982720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765309.482138:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765310.506274:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765311.529982:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765312.553972:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765313.578007:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765313.833970:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765313.833984:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765313.833989:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765313.834003:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00080000:0.0:1789765313.834017:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765313.834026:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765313.834077:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067982848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765313.834109:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765313.834113:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765313.834115:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765313.834119:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00080000:0.0:1789765313.834123:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765313.834126:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765313.834138:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765313.834165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067982976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765313.834177:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765313.834181:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765313.834184:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789765313.834193:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067982848:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765313.834194:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00080000:0.0:1789765313.834203:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789765313.834275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765313.834315:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020dac700 x1876702067982848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765313.834334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067982848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765313.834348:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765313.834353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765313.834357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067982848:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765313.834379:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765313.834401:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765313.834409:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765313.834417:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020daf100 x1876702067982976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789765313.834418:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765313.834441:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067982976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765313.834450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765313.834456:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765313.834457:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067983104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765313.834463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067982976:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765313.834489:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789765313.834494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983104:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765313.834497:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765313.834504:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765313.834511:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00080000:0.0:1789765313.834517:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765313.834522:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765313.834528:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789765313.834532:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765313.834549:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067983232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765313.834554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020dac380 x1876702067983104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765313.834596:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789765313.834605:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067983104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765313.834606:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765313.834613:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765313.834625:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00080000:0.0:1789765313.834639:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765313.834640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765313.834651:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789765313.834652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765313.834664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983104:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789765313.834665:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765313.834689:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067983360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765313.834690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983232:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765313.834715:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765313.834720:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765313.834722:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765313.834731:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2781 00000800:00000100:2.0:1789765313.834733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765313.834739:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765313.834746:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765313.834752:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765313.834755:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020dad180 x1876702067983232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765313.834768:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067983488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765313.834774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067983232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765313.834845:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765313.834848:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765313.834851:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765313.834857:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765313.834863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765313.834870:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2781 00000100:00100000:2.0:1789765313.834872:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983232:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765313.834878:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765313.834886:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765313.834892:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765313.834896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765313.834910:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067983616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789765313.834929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765313.834950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020dae300 x1876702067983360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765313.834963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067983360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765313.834971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765313.834974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765313.834977:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765313.834983:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067983488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765313.835055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765313.835067:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020dace00 x1876702067983488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765313.835078:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067983488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765313.835085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765313.835089:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765313.835092:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067983488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765313.835105:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067983616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765313.835125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765313.835136:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765313/real 1789765313] req@ffff936020daea00 x1876702067983616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765313.835159:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067983616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765313.835172:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765313.835176:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765313.835179:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067983616:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765314.602180:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765315.625992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765316.650131:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765317.675356:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765318.698154:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765318.956942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.956951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765318.956955:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.956964:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.956974:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.956982:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765318.957031:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067983744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765318.957058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765318.957064:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.957068:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957072:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.957076:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765318.957085:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765318.957097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067983872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765318.957116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765318.957121:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.957125:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957129:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.957133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765318.957138:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765318.957152:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067983744:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765318.957154:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067984000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765318.957170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765318.957175:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.957179:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957183:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.957188:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765318.957193:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765318.957205:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067984128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765318.957213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765318.957218:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.957221:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957225:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789765318.957228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765318.957232:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765318.957237:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765318.957249:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067984256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765318.957257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765318.957262:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765318.957265:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957269:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.957273:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957278:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef7800 x1876702067983744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789765318.957280:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765318.957296:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067984384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765318.957303:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067983744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765318.957310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765318.957319:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765318.957321:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765318.957324:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765318.957327:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2786 00000100:00080000:1.0:1789765318.957330:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765318.957336:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:2.0:1789765318.957338:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:1.0:1789765318.957343:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765318.957345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067983744:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765318.957357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067984512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765318.957357:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765318.957389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957403:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef6680 x1876702067983872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067983872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957425:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957428:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957432:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067983872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765318.957454:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765318.957474:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957485:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef7480 x1876702067984000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067984000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957518:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957522:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765318.957528:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765318.957547:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957557:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef7b80 x1876702067984128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957569:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067984128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765318.957589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765318.957615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef6300 x1876702067984256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067984256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765318.957655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067984384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765318.957672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957681:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef6d80 x1876702067984384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067984384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957704:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067984384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765318.957710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067984512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765318.957727:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765318.957738:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765318/real 1789765318] req@ffff936010ef4e00 x1876702067984512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765318.957749:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067984512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765318.957760:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765318.957763:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765318.957767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067984512:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765319.723175:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765320.746575:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765321.770011:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765322.793990:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765323.817960:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765324.073919:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.073928:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765324.073932:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.073942:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.073951:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.073959:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765324.073992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067984640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.074033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.074037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765324.074039:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.074043:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.074047:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.074050:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.074059:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765324.074086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067984640:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765324.074094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067984768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.074106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.074110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765324.074113:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.074117:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.074122:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.074126:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.074141:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765324.074161:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067984896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.074170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789765324.074207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.074245:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef7100 x1876702067984640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.074262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067984640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.074325:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.074329:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.074333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067984640:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765324.074353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765324.074373:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.074383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef4380 x1876702067984768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.074393:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067984768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.074400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.074403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.074406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765324.074412:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765324.074426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.074434:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef4000 x1876702067984896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.074444:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067984896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.074451:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.074454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.074456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067984896:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765324.075446:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765324.075449:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.075455:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.075461:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.075465:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.075471:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765324.075489:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067985024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.075512:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.075515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765324.075516:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.075520:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.075523:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.075526:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.075529:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765324.075538:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067985152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.075546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.075549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765324.075550:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.075553:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.075555:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.075558:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.075561:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765324.075568:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067985280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765324.075575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765324.075578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765324.075579:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765324.075582:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2791 00000100:00080000:1.0:1789765324.075584:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765324.075587:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765324.075590:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765324.075597:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067985408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765324.075874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765324.075915:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.075930:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef5c00 x1876702067985024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.076036:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067985024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.076045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.076048:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.076052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765324.076058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789765324.076088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.076099:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef4700 x1876702067985152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.076109:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067985152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.076116:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.076119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.076122:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765324.076127:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067985280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765324.076390:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.076404:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef4a80 x1876702067985280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.076416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067985280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.076423:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.076427:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.076430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067985280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765324.076435:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067985408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765324.076455:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765324.076465:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765324/real 1789765324] req@ffff936010ef5880 x1876702067985408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765324.076475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067985408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765324.076488:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765324.076491:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789765324.076494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067985408:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765324.842201:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765325.866942:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765326.893607:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765327.914071:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765328.938486:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765329.193925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.193934:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765329.193939:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.193948:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.193959:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.193967:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765329.194002:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702067985536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765329.194055:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194060:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194064:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194069:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194079:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765329.194098:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067985536:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765329.194223:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702067985664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765329.194252:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194305:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194310:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194315:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194320:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765329.194334:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702067985792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194354:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765329.194356:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194360:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194363:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194367:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194371:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765329.194381:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702067985920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765329.194394:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194398:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194401:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194404:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194408:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765329.194418:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702067986048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194427:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765329.194432:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194435:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194439:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194444:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194448:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765329.194458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765329.194460:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702067986176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765329.194474:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765329.194477:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765329.194479:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765329.194483:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2797 00000100:00080000:1.0:1789765329.194488:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765329.194492:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765329.194497:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765329.194502:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765329.194511:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702067986304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789765329.194532:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765329.194560:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765329.194582:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067986176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765329.194599:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067986304:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765329.961989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765330.986611:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765332.010552:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765332.266349:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765332.266408:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765332.266429:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266479:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266489:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266498:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266506:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266515:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765332.266524:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765332.268679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef4000 x1876702067986176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.268714:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702067986176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.268728:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.268734:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.268738:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067986176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765332.268746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef4380 x1876702067986304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.268759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702067986304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.268768:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.268772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.268775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067986304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765332.268784:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef5880 x1876702067985664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.268796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702067985664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.268838:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.268842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.268846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765332.269094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef4a80 x1876702067985792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.269108:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702067985792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.269115:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.269119:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.269123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765332.269131:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef4700 x1876702067985920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.269141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702067985920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.269149:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.269153:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.269156:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067985920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765332.269164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef5c00 x1876702067986048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.269176:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702067986048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.269183:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.269186:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.269189:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765332.269197:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765329/real 1789765332] req@ffff936010ef5180 x1876702067985536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765332.269210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702067985536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765332.269218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765332.269221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765332.269223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067985536:192.168.203.137@tcp:250:kworker.0 00010000:00010000:0.0:1789765333.034615:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765334.058224:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765334.314230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314240:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765334.314243:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314253:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314263:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314270:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765334.314302:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067986432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314321:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314324:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765334.314326:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314330:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314334:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314338:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314346:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765334.314358:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067986560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314367:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314370:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765334.314372:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314375:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314378:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314382:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314387:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765334.314398:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067986688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314406:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314409:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765334.314411:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314415:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314418:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314421:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314425:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765334.314435:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067986816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314443:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314445:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765334.314447:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314450:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314452:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314455:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314459:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765334.314468:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067986944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314475:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314479:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765334.314480:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314483:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314487:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314490:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314494:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765334.314502:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067987072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765334.314510:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765334.314513:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765334.314514:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765334.314517:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2802 00000100:00080000:3.0:1789765334.314520:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765334.314523:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765334.314527:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765334.314535:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067987200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765334.314690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067986432:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765334.314749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.314794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa6300 x1876702067986432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.314842:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067986432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.314852:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.314855:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.314859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067986432:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765334.314866:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765334.314888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.314900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa7b80 x1876702067986560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.314912:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067986560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.314919:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.314922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.314925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765334.314931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765334.314951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.314962:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa4000 x1876702067986688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.314971:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067986688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.314978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.314981:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.314984:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765334.315000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765334.315020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.315031:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa7800 x1876702067986816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.315041:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067986816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.315048:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.315050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.315053:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765334.315058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765334.315075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.315085:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa5c00 x1876702067986944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.315095:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067986944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.315102:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.315105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.315107:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067986944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765334.315114:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765334.315132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.315142:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa5880 x1876702067987072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.315152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067987072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.315160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.315163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.315165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765334.315170:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765334.315186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765334.315196:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765334/real 1789765334] req@ffff936010aa4a80 x1876702067987200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765334.315206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067987200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765334.315212:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765334.315215:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765334.315217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987200:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765335.082364:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765336.106703:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765337.136320:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765338.160333:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765339.178115:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765339.435623:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435634:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765339.435639:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435650:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435665:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435675:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765339.435717:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067987328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.435752:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435756:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765339.435758:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435762:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435766:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435770:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.435781:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765339.435793:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067987456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.435831:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435834:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765339.435836:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435840:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435844:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435847:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.435852:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765339.435864:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067987584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.435874:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435877:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765339.435879:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435883:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435886:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435890:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.435895:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765339.435905:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067987712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.435913:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435916:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765339.435917:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435921:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435924:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435927:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.435931:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765339.435941:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067987840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.435949:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.435963:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765339.435965:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.435969:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.435972:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.435976:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.435982:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765339.435993:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067987968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765339.436002:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765339.436005:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765339.436007:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765339.436010:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2807 00000100:00080000:0.0:1789765339.436013:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765339.436016:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765339.436031:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765339.436041:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067988096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765339.438065:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067987328:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765339.438175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.438233:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020daf800 x1876702067987328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.438253:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067987328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.438270:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.438274:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.438278:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067987328:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765339.438286:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765339.438309:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.438321:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020dadc00 x1876702067987456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.438331:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067987456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.438338:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.438342:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.438345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765339.438352:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765339.438371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.438381:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020dadf80 x1876702067987584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.438391:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067987584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.438398:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.438401:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.438404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765339.438411:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765339.438432:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.438454:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020daea00 x1876702067987712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.438700:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067987712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.438708:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.438712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.438715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765339.438721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765339.438745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.438755:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020dace00 x1876702067987840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.438766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067987840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.438774:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.438778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.438781:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067987840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765339.438787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765339.439037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.439049:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020dae300 x1876702067987968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.439062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067987968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.439069:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.439072:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.439076:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067987968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765339.439082:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765339.439106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765339.439117:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765339/real 1789765339] req@ffff936020dad180 x1876702067988096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765339.439129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067988096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765339.439137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765339.439140:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765339.439143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988096:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765340.202356:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765341.226886:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765342.250004:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765343.274013:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765344.298062:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765344.692682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765344.692691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765344.692695:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765344.692705:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:1.0:1789765344.692714:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765344.692721:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765344.692840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702067988224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765344.692869:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765344.692873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765344.692875:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765344.692879:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:1.0:1789765344.692882:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765344.692886:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765344.692908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765344.692942:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702067988352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765344.692954:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765344.692958:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765344.692960:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765344.692964:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:1.0:1789765344.692967:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765344.692971:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765344.693006:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067988224:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765344.693064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.693102:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936010ef7100 x1876702067988224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.693158:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702067988224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.693175:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.693179:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.693183:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067988224:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765344.693190:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765344.693213:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.693225:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936010ef4e00 x1876702067988352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.693236:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702067988352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.693243:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.693246:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.693249:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988352:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789765344.693737:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765344.693755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067988480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765344.694153:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765344.694637:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765344.695876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765344.695882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765344.695885:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765344.695891:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:0.0:1789765344.695897:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765344.695903:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765344.695912:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765344.696960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936020dac380 x1876702067988480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.697125:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067988480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.697137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.697141:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.697145:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765344.697216:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067988608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765344.697229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765344.697233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765344.697235:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765344.697239:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:0.0:1789765344.697243:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765344.697248:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765344.697254:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765344.697262:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067988736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765344.697272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765344.697275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765344.697277:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765344.697280:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:0.0:1789765344.697283:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765344.697287:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765344.697291:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765344.697294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765344.697302:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067988864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765344.697313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765344.697316:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765344.697318:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765344.697322:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2812 00000100:00080000:0.0:1789765344.697326:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765344.697330:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765344.697334:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765344.697345:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067988992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789765344.697354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.697372:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936020daf100 x1876702067988608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.697384:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067988608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.697392:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.697395:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.697398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765344.697404:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765344.697423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.697434:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936020dac700 x1876702067988736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.697446:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067988736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.697454:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.697457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.697460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067988736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765344.697475:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765344.697495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.697506:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936020daed80 x1876702067988864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.697517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067988864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.697524:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.697527:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.697530:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765344.697536:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765344.697555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765344.698147:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765344/real 1789765344] req@ffff936020dae680 x1876702067988992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765344.698164:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067988992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765344.698235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765344.698238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765344.698242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067988992:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765345.324976:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765346.349311:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765347.381092:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765348.395950:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765349.426224:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765350.186384:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186400:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765350.186405:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186414:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186426:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186432:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765350.186469:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067989120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186500:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186504:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765350.186506:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186510:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186514:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186517:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186525:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765350.186536:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067989248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186545:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186548:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765350.186550:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186553:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186556:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186559:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186563:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765350.186570:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067989376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186578:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186581:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765350.186583:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186586:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186590:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186593:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186597:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765350.186606:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067989504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186614:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186617:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765350.186619:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186622:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186625:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186638:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186642:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765350.186651:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067989632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186660:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186663:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765350.186665:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186668:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186671:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186674:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186677:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765350.186684:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067989760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765350.186699:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765350.186702:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765350.186703:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765350.186706:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2817 00000100:00080000:3.0:1789765350.186709:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765350.186712:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765350.186715:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765350.186723:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067989888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765350.187130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067989120:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765350.187198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187244:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa4380 x1876702067989120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187260:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067989120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187270:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187273:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187278:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067989120:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765350.187285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765350.187305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187315:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa4e00 x1876702067989248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067989248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187342:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187345:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187348:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765350.187353:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765350.187370:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187379:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa5f80 x1876702067989376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067989376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187525:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765350.187530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765350.187551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa7480 x1876702067989504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187573:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067989504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187592:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765350.187597:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765350.187615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa6680 x1876702067989632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067989632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187909:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187913:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067989632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765350.187922:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067989760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765350.187945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.187955:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa5180 x1876702067989760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.187966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067989760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.187974:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.187977:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.187980:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067989760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765350.187986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067989888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765350.188002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765350.188012:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765350/real 1789765350] req@ffff936010aa4700 x1876702067989888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765350.188022:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067989888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765350.188030:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765350.188033:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765350.188036:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067989888:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765350.442365:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765351.468344:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765352.490490:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765353.514043:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765354.538196:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765355.308667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.308677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765355.308681:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.308690:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.308701:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.308709:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765355.308743:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067990016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.308777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.308783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765355.308785:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.308788:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.308792:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.308796:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.308852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765355.308865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067990144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.308874:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.308877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765355.308879:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.308883:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.308887:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.308890:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.308895:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765355.308905:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067990272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.308929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.308932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765355.308934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.308937:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.308941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.308945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.308950:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765355.308959:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067990400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.308968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.308971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765355.308972:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.308976:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.308979:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.308982:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.308986:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765355.308996:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067990528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.309005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.309008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765355.309010:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.309013:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.309016:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.309019:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.309023:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765355.309034:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067990656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765355.309196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765355.309199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765355.309202:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765355.309206:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2823 00000100:00080000:0.0:1789765355.309209:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765355.309213:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765355.309218:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765355.309229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067990784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765355.309822:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067990016:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765355.309892:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.309949:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020dac000 x1876702067990016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.309970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067990016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.309979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.309983:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.309987:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067990016:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765355.309994:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765355.310017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310030:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020dad880 x1876702067990144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067990144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310059:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310063:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310066:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765355.310072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765355.310091:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020dad500 x1876702067990272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310112:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067990272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310128:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310130:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765355.310136:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765355.310153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020daf480 x1876702067990400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310174:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067990400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310181:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310184:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310186:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765355.310191:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765355.310208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020dafb80 x1876702067990528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067990528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067990528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765355.310248:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067990656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765355.310266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020dae680 x1876702067990656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067990656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310297:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310299:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310302:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067990656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765355.310307:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067990784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765355.310326:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765355.310335:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765355/real 1789765355] req@ffff936020daed80 x1876702067990784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765355.310344:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067990784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765355.310351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765355.310353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765355.310356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067990784:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765355.565695:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765356.586531:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765357.610032:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765358.634872:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765359.659033:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765360.427255:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427266:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765360.427269:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427279:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427290:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427297:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765360.427335:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067990912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427368:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427372:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765360.427374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427379:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427384:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427388:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427397:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765360.427409:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067991040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427419:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427422:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765360.427423:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427427:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427431:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427434:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427438:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765360.427450:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067991168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427459:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427461:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765360.427463:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427466:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427469:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427473:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765360.427487:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067991296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427495:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427498:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765360.427499:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427503:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427507:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427510:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427514:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765360.427523:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067991424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427530:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427532:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765360.427534:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427537:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427540:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427543:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427547:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765360.427556:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067991552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765360.427563:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765360.427566:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765360.427567:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765360.427570:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2828 00000100:00080000:0.0:1789765360.427573:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765360.427576:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765360.427579:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765360.427587:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067991680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765360.427726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067990912:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765360.427787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.427867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020dac700 x1876702067990912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.427883:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067990912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.427893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.427897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.427900:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067990912:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765360.427907:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765360.427929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.427941:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020daf100 x1876702067991040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.427953:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067991040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.427960:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.427963:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.427966:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765360.427972:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765360.427991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.428002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020dac380 x1876702067991168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.428020:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067991168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.428027:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.428030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.428034:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765360.428040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765360.428061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.428072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020dad180 x1876702067991296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.428084:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067991296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.428091:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.428094:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.428616:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765360.428623:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765360.428650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.428662:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020dae300 x1876702067991424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.428674:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067991424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.428687:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.428690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.428693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765360.428699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067991552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765360.428720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.428733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020dace00 x1876702067991552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.428746:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067991552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.428755:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.428759:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.428762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067991552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765360.428770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067991680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765360.428788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765360.428828:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765360/real 1789765360] req@ffff936020daea00 x1876702067991680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765360.428841:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067991680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765360.428849:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765360.428852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765360.428855:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067991680:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765360.682031:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765361.707985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765362.732035:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765363.756746:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765364.777983:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765365.545913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.545921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765365.545924:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.545932:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.545941:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.545946:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765365.545977:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702067991808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546004:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765365.546008:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546011:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546013:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546015:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546022:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765365.546029:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702067991936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765365.546038:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546040:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546043:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546045:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546047:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765365.546052:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702067992064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765365.546060:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546063:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546064:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546066:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546068:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765365.546073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702067992192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765365.546080:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546082:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546083:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546085:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546088:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765365.546092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702067992320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765365.546104:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546105:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546108:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546110:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765365.546114:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702067992448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765365.546119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765365.546121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765365.546121:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765365.546123:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2833 00000100:00080000:0.0:1789765365.546124:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765365.546126:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765365.546129:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765365.546133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702067992576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765365.546209:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067991808:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765365.546253:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020dadf80 x1876702067991808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546291:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702067991808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546298:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546300:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546303:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067991808:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765365.546307:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765365.546318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546325:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020dadc00 x1876702067991936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546332:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702067991936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546338:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546340:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067991936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765365.546343:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765365.546354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546360:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020daf800 x1876702067992064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702067992064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546371:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546374:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765365.546377:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765365.546386:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546391:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020daea00 x1876702067992192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546396:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702067992192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765365.546407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765365.546419:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546428:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020dace00 x1876702067992320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546444:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702067992320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546452:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546457:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765365.546463:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067992448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765365.546482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020dae300 x1876702067992448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702067992448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546513:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546516:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067992448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765365.546521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067992576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765365.546539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765365.546555:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765365/real 1789765365] req@ffff936020dad180 x1876702067992576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765365.546565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702067992576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765365.546578:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765365.546581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765365.546584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067992576:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765365.802127:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765366.825946:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765367.849999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765368.874137:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765369.898457:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765370.665886:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.665897:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765370.665902:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.665912:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.665922:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.665929:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765370.666023:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702067992704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666056:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666060:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765370.666062:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666066:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666070:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666073:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666082:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765370.666093:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702067992832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666102:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666105:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765370.666107:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666111:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666114:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666118:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666122:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765370.666132:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702067992960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666147:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666150:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765370.666151:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666155:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666158:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666161:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666165:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765370.666173:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067993088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666182:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666185:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765370.666187:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666190:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666192:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666195:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666199:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765370.666206:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067993216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666217:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765370.666219:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666222:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666225:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666228:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765370.666240:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067993344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765370.666248:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765370.666251:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765370.666253:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765370.666256:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2838 00000100:00080000:0.0:1789765370.666259:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765370.666262:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765370.666266:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765370.666275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067993472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765370.666367:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067992704:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765370.666420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666468:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020dac380 x1876702067992704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666484:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702067992704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666493:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666496:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666499:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067992704:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765370.666506:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765370.666527:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666538:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020daf100 x1876702067992832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702067992832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666562:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765370.666568:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765370.666600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666611:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020dac700 x1876702067992960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702067992960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666629:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067992960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765370.666641:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765370.666659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666669:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020daed80 x1876702067993088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666679:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067993088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765370.666696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765370.666712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666740:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020dae680 x1876702067993216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067993216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765370.666770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067993344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765370.666787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666821:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020dafb80 x1876702067993344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666834:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067993344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666848:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067993344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765370.666853:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067993472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765370.666870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765370.666879:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765370/real 1789765370] req@ffff936020daf480 x1876702067993472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765370.666889:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067993472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765370.666896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765370.666899:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765370.666903:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067993472:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765370.924296:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765371.946408:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765372.970192:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765373.994396:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765375.017989:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765375.787905:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765375.787915:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765375.787920:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.787929:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.787940:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.787946:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765375.787981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702067993600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765375.788019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765375.788023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765375.788024:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788028:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.788032:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.788036:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765375.788044:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765375.788055:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702067993728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765375.788064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765375.788066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765375.788077:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788082:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.788086:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.788090:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765375.788094:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765375.788102:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067993600:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765375.788108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702067993856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765375.788119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765375.788122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765375.788124:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788127:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.788130:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.788135:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765375.788139:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765375.788150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702067993984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765375.788158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789765375.788164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765375.788165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765375.788169:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788173:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.788177:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.788181:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765375.788186:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765375.788197:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702067994112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765375.788208:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061dce00 x1876702067993600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702067993600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788241:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067993600:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765375.788251:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765375.788271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061dc000 x1876702067993728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788293:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702067993728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765375.788327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765375.788344:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788353:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061dd500 x1876702067993856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702067993856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765375.788384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993984:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765375.788395:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765375.788399:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765375.788401:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788406:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2843 00000800:00000100:1.0:1789765375.788410:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765375.788415:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765375.788423:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788425:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061df480 x1876702067993984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789765375.788430:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765375.788443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702067993984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765375.788451:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067994240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765375.788455:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789765375.788470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765375.788471:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067993984:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765375.788478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765375.788481:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765375.788486:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2843 00000100:00080000:2.0:1789765375.788489:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789765375.788490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994112:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765375.788496:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765375.788501:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765375.788510:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067994368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789765375.788529:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788542:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061df100 x1876702067994112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788555:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702067994112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788563:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788566:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765375.788575:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067994240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765375.788593:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788603:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061dd880 x1876702067994240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067994240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788625:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788628:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067994240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765375.788640:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067994368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765375.788658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765375.788679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765375/real 1789765375] req@ffff9360061ddc00 x1876702067994368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765375.788707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067994368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765375.788715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765375.788718:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765375.788721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067994368:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765376.045896:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765377.067578:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765378.090847:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765379.114057:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765380.138066:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765380.906046:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.906056:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765380.909948:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.909970:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.909980:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.909987:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765380.910022:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702067994496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910040:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910044:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765380.910046:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910050:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910053:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910056:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910161:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765380.910184:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702067994624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910195:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910198:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765380.910200:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910204:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910207:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910211:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910215:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765380.910225:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702067994752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910234:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910237:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765380.910239:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910242:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910245:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910248:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910251:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765380.910260:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702067994880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910268:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910271:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765380.910273:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910276:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910279:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910282:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910286:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765380.910295:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702067995008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910303:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910306:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765380.910308:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910311:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910313:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910316:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910320:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765380.910327:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702067995136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765380.910335:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765380.910338:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765380.910340:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765380.910342:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2848 00000100:00080000:0.0:1789765380.910345:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765380.910347:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765380.910351:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765380.910359:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702067995264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765380.910528:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067994496:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765380.910583:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.910634:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020dad500 x1876702067994496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.910657:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702067994496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.910666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.910670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.910673:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067994496:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765380.910680:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765380.910699:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.910710:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020dad880 x1876702067994624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.910721:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702067994624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.910729:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.910854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.910858:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765380.910864:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765380.910884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.910895:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020dac000 x1876702067994752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.910906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702067994752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.910914:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.910917:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.910920:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765380.910926:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765380.910941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.910950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020daf480 x1876702067994880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.911295:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702067994880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.911304:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.911307:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.911310:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067994880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765380.911317:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765380.911339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.911350:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020dafb80 x1876702067995008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.911362:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702067995008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.911370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.911373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.911376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765380.911382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067995136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765380.911398:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.911408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020dae680 x1876702067995136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.911419:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702067995136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.911426:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.911429:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.911432:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067995136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765380.911438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067995264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765380.911452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765380.911461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765380/real 1789765380] req@ffff936020daed80 x1876702067995264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765380.911471:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702067995264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765380.911478:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765380.911481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765380.911484:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067995264:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765381.161993:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765382.186435:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765383.210420:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765384.234197:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765385.258038:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765386.027680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.027689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765386.027701:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765386.027712:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.027721:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765386.027728:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765386.027774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067995392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765386.027904:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067995392:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765386.027962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765386.028000:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061de300 x1876702067995392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765386.028254:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.028259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765386.028261:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765386.028265:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.028270:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765386.028273:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765386.028281:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765386.028301:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067995392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765386.028301:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702067995520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765386.028335:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.028339:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765386.028353:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765386.028357:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.028361:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765386.028363:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765386.028367:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765386.028378:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702067995648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765386.028386:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765386.028388:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765386.028389:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765386.028393:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765386.028398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789765386.028399:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.028404:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789765386.028407:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067995392:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765386.028411:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765386.028419:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765386.028429:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702067995776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765386.028437:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995520:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765386.028441:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.028445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765386.028447:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765386.028451:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.028455:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765386.028459:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765386.028464:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789765386.028469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765386.028478:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702067995904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765386.028485:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061ddf80 x1876702067995520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765386.028488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.028497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765386.028500:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789765386.028510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702067995520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765386.028518:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:1.0:1789765386.028522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765386.028524:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765386.028528:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789765386.028532:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765386.028536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995520:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789765386.028539:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765386.028544:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789765386.028553:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702067996032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765386.028563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765386.028567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:1.0:1789765386.028571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765386.028571:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765386.028578:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2853 00000100:00080000:2.0:1789765386.028583:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789765386.028588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061dc700 x1876702067995648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765386.028588:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765386.028597:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765386.028609:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702067995648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765386.028622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702067996160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765386.028634:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765386.028640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765386.028645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765386.028663:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765386.032300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765386.032337:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061dc380 x1876702067995776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765386.032356:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702067995776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765386.032368:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765386.032372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765386.032377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765386.032387:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765386.032420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765386.032431:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061de300 x1876702067995904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765386.032442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702067995904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765386.032450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765386.032452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765386.032456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067995904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765386.032462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765386.032482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765386.032493:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061ddc00 x1876702067996032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765386.032503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702067996032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765386.032511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765386.032514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765386.032517:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765386.032523:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765386.032863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765386.032877:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765386/real 1789765386] req@ffff9360061dd880 x1876702067996160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765386.032889:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702067996160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765386.032896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765386.032900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765386.032903:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996160:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765386.282479:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765387.307270:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765388.330075:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765389.354466:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765390.378052:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765391.146136:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146146:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765391.146150:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146160:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146169:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146176:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765391.146329:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702067996288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146362:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146366:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765391.146368:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146372:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146377:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146381:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146391:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765391.146404:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702067996416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146413:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146427:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765391.146429:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146433:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146437:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146440:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146444:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765391.146456:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702067996544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146464:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146467:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765391.146469:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146472:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146475:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146478:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146482:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765391.146493:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702067996672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146501:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146504:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765391.146505:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146509:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146512:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146515:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146519:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765391.146528:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702067996800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146536:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146539:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765391.146540:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146543:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146546:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146550:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146554:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765391.146563:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702067996928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765391.146571:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765391.146584:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765391.146585:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765391.146589:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2858 00000100:00080000:3.0:1789765391.146593:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765391.146597:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765391.146601:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765391.146611:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702067997056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765391.146702:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067996288:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765391.146763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.146819:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa7100 x1876702067996288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.146845:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702067996288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.146855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.146858:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.146862:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067996288:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765391.146868:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765391.146887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.146897:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa5500 x1876702067996416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.146913:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702067996416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.146921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.146924:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.146927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765391.146931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765391.146945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.146953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa6a00 x1876702067996544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.146964:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702067996544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.146971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.146973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.146976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765391.146981:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765391.146994:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.147002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa4700 x1876702067996672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.147011:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702067996672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.147024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.147027:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.147030:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765391.147034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765391.147047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.147055:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa5180 x1876702067996800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.147064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702067996800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.147071:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.147073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.147076:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067996800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765391.147080:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765391.147094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.147101:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa6680 x1876702067996928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.147121:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702067996928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.147129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.147132:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.147135:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067996928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765391.147140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765391.147154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765391.147162:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765391/real 1789765391] req@ffff936010aa7480 x1876702067997056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765391.147173:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702067997056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765391.147179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765391.147182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765391.147184:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997056:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765391.402966:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765392.428313:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765393.450079:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765394.474131:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765395.498206:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765396.265900:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.265910:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765396.265914:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.265923:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.265934:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.265942:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765396.265981:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702067997184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266015:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266019:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765396.266020:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266024:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266028:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266033:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266042:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765396.266054:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702067997312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266063:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266066:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765396.266068:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266072:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266075:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266079:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266083:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765396.266091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702067997440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266100:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266103:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765396.266104:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266462:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266467:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266472:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765396.266489:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067997568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266498:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266501:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765396.266503:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266506:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266509:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266512:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266516:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765396.266523:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067997696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266531:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266534:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765396.266544:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266547:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266550:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266553:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266557:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765396.266568:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067997824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765396.266576:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765396.266579:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765396.266581:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765396.266583:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2864 00000100:00080000:3.0:1789765396.266586:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765396.266589:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765396.266593:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765396.266600:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067997952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765396.266840:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067997184:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765396.266922:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765396.266948:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765396.266968:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765396.266988:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765396.267005:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765396.267022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997952:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765396.525291:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765397.546139:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765398.570354:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789765399.341569:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789765399.341611:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789765399.341627:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341665:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341673:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341681:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341689:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341698:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789765399.341706:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765399.341853:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa5f80 x1876702067997184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.341877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702067997184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.341888:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.341893:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.341897:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067997184:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765399.341905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa4e00 x1876702067997312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.341915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702067997312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.341922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.341925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.341928:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765399.341934:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa4380 x1876702067997440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.341946:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702067997440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.341953:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.341956:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.341959:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765399.341966:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa4a80 x1876702067997568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.341978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067997568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.341986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.341989:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.341992:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765399.341997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa5880 x1876702067997696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.342009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067997696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.342016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.342019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.342022:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067997696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765399.342028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa5c00 x1876702067997824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.342039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067997824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.342047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.342050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.342053:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765399.342059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765396/real 1789765399] req@ffff936010aa7800 x1876702067997952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765399.342069:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067997952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765399.342076:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765399.342080:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765399.342083:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067997952:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765399.594792:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765400.620055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765401.388671:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388680:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765401.388684:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388694:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388704:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388711:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765401.388742:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702067998080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388772:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388775:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765401.388777:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388781:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388785:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388788:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388813:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765401.388826:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702067998208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388836:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388839:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765401.388841:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388844:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388848:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388851:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388855:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765401.388864:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702067998336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388873:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388875:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765401.388877:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388880:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388883:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388886:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388890:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765401.388901:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702067998464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388908:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388911:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765401.388913:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388916:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388919:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388922:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388926:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765401.388935:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702067998592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388942:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388945:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765401.388947:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388950:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388953:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388956:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388960:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765401.388969:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702067998720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765401.388977:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765401.388980:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765401.388981:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765401.388984:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2869 00000100:00080000:3.0:1789765401.388987:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765401.388990:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765401.388993:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765401.389002:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702067998848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765401.389514:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067998080:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765401.389576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa4000 x1876702067998080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702067998080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389638:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389644:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067998080:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765401.389651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765401.389672:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389683:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa7b80 x1876702067998208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389693:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702067998208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389710:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765401.389718:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765401.389736:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa6300 x1876702067998336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389757:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702067998336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389765:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389768:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389771:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765401.389777:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765401.389822:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389837:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa7800 x1876702067998464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389848:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702067998464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389856:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389858:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389862:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765401.389868:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765401.389888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389898:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa5c00 x1876702067998592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389908:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702067998592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389917:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389923:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067998592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765401.389928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067998720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765401.389946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.389957:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa5880 x1876702067998720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.389966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702067998720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.389974:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.389976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.389979:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067998720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765401.389984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067998848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765401.390002:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765401.390013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765401/real 1789765401] req@ffff936010aa4a80 x1876702067998848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765401.390024:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702067998848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765401.390032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765401.390034:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765401.390037:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067998848:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765401.643585:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765402.666347:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765403.690037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765404.714525:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765405.741751:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765406.506023:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506032:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765406.506037:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506046:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506117:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506129:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765406.506164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702067998976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506195:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506199:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765406.506201:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506205:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506208:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506211:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765406.506238:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702067999104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506247:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765406.506252:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506256:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506259:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506262:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506266:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765406.506275:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702067999232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506283:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506286:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765406.506288:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506291:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506294:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506297:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506300:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765406.506309:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702067999360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506316:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506319:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765406.506321:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506325:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506328:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506331:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506335:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765406.506344:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702067999488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506352:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506355:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765406.506356:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506360:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506363:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506366:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506370:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765406.506378:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702067999616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765406.506386:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765406.506389:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765406.506391:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765406.506394:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2874 00000100:00080000:1.0:1789765406.506396:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765406.506399:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765406.506403:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765406.506411:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702067999744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765406.506611:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067998976:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765406.506668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.506712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef6d80 x1876702067998976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.506730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702067998976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.506740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.506744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.506747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067998976:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765406.506915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765406.506939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.506950:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef6300 x1876702067999104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.506962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702067999104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.506970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.506973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.506976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765406.506981:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765406.506998:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.507008:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef7b80 x1876702067999232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.507019:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702067999232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.507026:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.507029:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.507032:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765406.507038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765406.507127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.507139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef7480 x1876702067999360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.507151:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702067999360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.507159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.507162:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.507165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765406.507171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765406.507188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.507197:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef6680 x1876702067999488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.507208:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702067999488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.507222:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.507225:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.507228:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702067999488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765406.507234:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067999616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765406.507250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.507259:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef7800 x1876702067999616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.507271:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702067999616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.507278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.507282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.507284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067999616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765406.507290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067999744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765406.507307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765406.507317:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765406/real 1789765406] req@ffff936010ef5500 x1876702067999744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765406.507327:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702067999744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765406.507335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765406.507337:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765406.507341:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702067999744:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765406.762112:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765407.788864:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765408.810245:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765409.834023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765410.864268:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765411.628788:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.628825:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765411.628830:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.628840:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.628849:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.628856:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765411.628893:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702067999872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.628914:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.628918:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765411.628919:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.628923:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.628926:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.628930:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.628940:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765411.628951:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068000000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.628960:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.628963:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765411.628964:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.628968:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.628971:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.628974:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.628978:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765411.628989:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068000128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.628997:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.629000:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765411.629002:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.629005:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.629008:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.629012:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.629016:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765411.629026:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068000256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.629034:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.629037:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765411.629038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.629041:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.629045:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.629054:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.629058:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765411.629066:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068000384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.629079:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.629082:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765411.629083:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.629086:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.629089:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.629093:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.629097:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765411.629106:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068000512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765411.629113:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765411.629116:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765411.629118:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765411.629122:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2879 00000100:00080000:1.0:1789765411.629125:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765411.629128:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765411.629132:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765411.629141:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068000640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765411.629554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067999872:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765411.629615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.629654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef6a00 x1876702067999872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.629672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702067999872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.629683:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.629686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.629690:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702067999872:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765411.629697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765411.629718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.629729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef5f80 x1876702068000000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.629741:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068000000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.629749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.629753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.629756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765411.629765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765411.629783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.629793:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef5500 x1876702068000128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.629833:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068000128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.629841:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.629844:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.629847:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765411.629854:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765411.629872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.629883:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef7800 x1876702068000256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.629894:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068000256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.629903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.629906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.629910:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765411.629915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765411.629932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.629942:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef6680 x1876702068000384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.629954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068000384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.629962:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.629965:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.629968:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765411.629974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068000512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765411.629992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.630002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef7480 x1876702068000512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.630012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068000512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.630019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.630023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.630026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068000512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765411.630031:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068000640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765411.630064:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765411.630075:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765411/real 1789765411] req@ffff936010ef7b80 x1876702068000640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765411.630086:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068000640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765411.630098:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765411.630102:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765411.630105:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068000640:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765411.882532:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765412.905895:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765413.932527:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765414.954244:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765415.978347:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765416.746065:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746075:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765416.746080:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.746089:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.746099:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.746106:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765416.746144:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068000768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765416.746177:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746181:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765416.746183:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.746197:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.746202:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.746206:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.746215:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765416.746227:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068000896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765416.746237:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746240:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765416.746241:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.746245:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.746248:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.746250:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.746255:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765416.746265:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068001024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765416.746279:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746294:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765416.746296:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.746300:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.746303:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.746307:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.746311:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765416.746322:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068001152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765416.746330:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746333:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765416.746335:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.746338:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.746341:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.746345:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.746348:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765416.746358:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068001280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765416.746366:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.746369:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789765416.746657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068000768:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765416.746714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765416.746764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa4380 x1876702068000768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765416.746778:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068000768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765416.746787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765416.746790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765416.746794:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068000768:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765416.746838:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765416.746862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765416.746874:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa4e00 x1876702068000896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765416.746885:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068000896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765416.746897:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765416.746900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765416.746904:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068000896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765416.746909:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765416.746929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765416.746940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa5f80 x1876702068001024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765416.747168:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068001024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765416.747177:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765416.747180:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765416.747183:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765416.747188:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765416.747210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765416.747221:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa7480 x1876702068001152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765416.747233:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068001152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765416.747241:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765416.747243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765416.747247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765416.747253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765416.747271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765416.747281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa6680 x1876702068001280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765416.747292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068001280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765416.747299:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765416.747301:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765416.747304:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001280:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765416.747828:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.747836:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.747842:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.747847:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.747855:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765416.747875:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068001408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765416.747930:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068001408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765416.748036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765416.748119:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765416.748123:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765416.748126:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765416.748130:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2884 00000100:00080000:3.0:1789765416.748134:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765416.748137:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765416.748142:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765416.748154:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068001536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765416.750568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa5180 x1876702068001408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765416.750592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068001408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765416.750604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765416.750608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765416.750612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068001408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765416.750644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068001536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765416.750695:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765416.750709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765416/real 1789765416] req@ffff936010aa4700 x1876702068001536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765416.750720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068001536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765416.750727:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765416.750730:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765416.750733:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068001536:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765417.002015:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765418.030435:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765419.050378:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765420.074078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765421.100322:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765421.865987:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.865997:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765421.866000:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866009:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866018:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866025:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765421.866061:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068001664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866099:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765421.866104:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866108:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866112:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866116:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866125:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765421.866138:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068001792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866147:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866150:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765421.866152:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866155:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866159:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866162:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866167:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765421.866176:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068001920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866185:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866188:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765421.866189:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866193:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866196:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866200:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866204:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765421.866212:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068002048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866220:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866223:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765421.866225:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866228:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866231:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866234:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866238:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765421.866246:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068002176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866254:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866257:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765421.866259:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866262:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866265:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866268:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866272:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765421.866279:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068002304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765421.866288:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765421.866291:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765421.866293:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765421.866296:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2889 00000100:00080000:1.0:1789765421.866299:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765421.866302:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765421.866305:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765421.866313:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068002432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765421.868389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068001664:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765421.868452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.868502:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef6300 x1876702068001664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.868519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068001664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.868529:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.868533:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.868537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068001664:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765421.868548:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765421.868571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.868587:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef6d80 x1876702068001792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.868599:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068001792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.868606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.868608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.868611:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765421.868627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765421.868646:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.868656:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef4e00 x1876702068001920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.868667:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068001920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.868674:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.868677:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.868679:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068001920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765421.868696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765421.868716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.869066:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef7100 x1876702068002048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.869146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068002048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.869154:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.869157:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.869160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765421.869167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765421.869189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.869201:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef5180 x1876702068002176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.869212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068002176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.869218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.869221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.869224:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765421.869229:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068002304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765421.869247:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.869256:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef5c00 x1876702068002304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.869265:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068002304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.869276:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.869279:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.869282:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068002304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765421.869287:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068002432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765421.869305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765421.869315:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765421/real 1789765421] req@ffff936010ef4700 x1876702068002432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765421.869326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068002432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765421.869332:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765421.869335:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765421.869338:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068002432:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765422.122248:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765423.146140:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765424.170021:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765425.196132:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765426.227542:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765426.985938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.985949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765426.985952:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.985962:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.985973:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.985980:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765426.986021:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068002560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.986989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.986994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765426.986997:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987188:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987196:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987201:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987211:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765426.987233:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068002688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.987251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.987255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765426.987256:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987261:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987264:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987268:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987272:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765426.987282:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068002816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.987291:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.987294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765426.987296:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987299:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987303:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987306:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987321:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765426.987332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068002944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.987340:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.987343:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765426.987345:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987348:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987352:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987355:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765426.987368:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068003072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.987376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.987379:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765426.987381:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987384:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987387:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987390:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987394:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765426.987402:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068003200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765426.987410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765426.987413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765426.987414:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765426.987417:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2894 00000100:00080000:2.0:1789765426.987420:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765426.987423:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765426.987427:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765426.987436:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068003328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765426.987521:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068002560:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765426.987576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061df100 x1876702068002560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068002560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987669:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987673:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987677:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068002560:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765426.987684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765426.987706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987717:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061df480 x1876702068002688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068002688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765426.987746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765426.987763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061dd500 x1876702068002816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068002816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987816:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987819:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987823:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765426.987828:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765426.987848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061dc000 x1876702068002944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987869:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068002944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987877:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987880:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987883:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068002944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765426.987888:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765426.987905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061dce00 x1876702068003072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068003072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987938:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765426.987944:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068003200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765426.987962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.987973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061dd180 x1876702068003200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.987984:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068003200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.987992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.987995:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.987998:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068003200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765426.988003:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068003328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765426.988021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765426.990376:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765426/real 1789765426] req@ffff9360061dca80 x1876702068003328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765426.990417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068003328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765426.990428:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765426.990433:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765426.990438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068003328:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765427.242192:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765428.265956:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765429.290639:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765430.314179:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765431.338098:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765432.106420:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106434:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765432.106439:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106452:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106466:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106476:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765432.106530:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068003456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106565:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106570:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765432.106572:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106576:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106594:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765432.106607:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068003584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765432.106623:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106627:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106630:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106633:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765432.106646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068003712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765432.106660:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106664:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106668:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106671:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106676:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765432.106687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068003840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765432.106701:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106704:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106707:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106710:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765432.106723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068003968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765432.106737:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106747:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765432.106758:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068004096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765432.106766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765432.106769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765432.106771:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765432.106775:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2899 00000100:00080000:3.0:1789765432.106777:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765432.106780:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765432.106785:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765432.106794:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068004224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765432.106986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068003456:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765432.107062:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa6a00 x1876702068003456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068003456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107156:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068003456:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765432.107167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765432.107190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa5500 x1876702068003584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107215:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068003584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107224:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765432.107235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765432.107256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107266:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa7100 x1876702068003712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068003712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107419:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107422:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107425:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765432.107432:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765432.107456:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107467:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa4700 x1876702068003840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107478:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068003840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107486:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107489:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107492:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765432.107498:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765432.107515:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107525:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa5180 x1876702068003968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107536:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068003968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107544:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107547:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068003968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765432.107556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765432.107576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa6680 x1876702068004096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107598:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068004096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107613:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765432.107618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765432.107638:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765432.107649:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765432/real 1789765432] req@ffff936010aa7480 x1876702068004224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765432.107662:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068004224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765432.107669:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765432.107672:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765432.107675:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004224:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765432.362131:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765433.386521:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765434.410393:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765435.434368:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765436.458427:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765437.226016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226027:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765437.226031:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226040:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226050:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226057:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765437.226094:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068004352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226124:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765437.226130:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226138:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226142:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226151:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765437.226161:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068004480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226175:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765437.226177:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226180:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226187:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765437.226202:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068004608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765437.226216:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226219:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226222:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226225:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765437.226249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068004736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226257:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765437.226261:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226264:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226267:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226270:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226274:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765437.226282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068004864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765437.226294:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226296:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226298:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226302:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765437.226314:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068004992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765437.226321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765437.226324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765437.226325:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765437.226328:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2905 00000100:00080000:2.0:1789765437.226330:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765437.226334:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765437.226337:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765437.226345:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068005120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765437.226459:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068004352:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765437.226514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226560:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061df800 x1876702068004352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068004352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226590:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068004352:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765437.226599:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765437.226621:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061dfb80 x1876702068004480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068004480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226659:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765437.226665:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765437.226682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061de680 x1876702068004608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068004608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765437.226725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765437.226743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226754:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061dca80 x1876702068004736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068004736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226779:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765437.226790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765437.226862:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226874:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061dd180 x1876702068004864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068004864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068004864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765437.226911:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765437.226928:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226938:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061dce00 x1876702068004992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.226950:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068004992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.226958:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.226962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.226965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068004992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765437.226971:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068005120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765437.226987:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765437.226997:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765437/real 1789765437] req@ffff9360061dc000 x1876702068005120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765437.227009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068005120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765437.227016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765437.227019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765437.227023:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068005120:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765437.482105:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765438.506174:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765439.530117:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765440.554000:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765441.578011:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765442.346608:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765442.346621:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346630:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346648:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765442.346682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068005248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346713:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765442.346718:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346729:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765442.346750:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068005376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346767:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765442.346768:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346772:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346774:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346778:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765442.346791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068005504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346842:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765442.346843:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346847:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346851:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346854:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346859:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765442.346870:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068005632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765442.346884:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346887:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346891:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346895:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346899:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765442.346909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068005760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765442.346921:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346924:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346926:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346929:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346933:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765442.346940:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068005888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765442.346946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765442.346949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765442.346951:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765442.346953:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2910 00000100:00080000:3.0:1789765442.346956:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765442.346959:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765442.346963:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765442.346970:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068006016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765442.347471:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068005248:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765442.347528:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.347619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa5f80 x1876702068005248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.347638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068005248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.347649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.347652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.347656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068005248:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765442.347663:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765442.347687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.347699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa4e00 x1876702068005376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.347709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068005376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.347715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.347718:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.347721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765442.347728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765442.347745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.347756:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa4380 x1876702068005504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.347765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068005504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.347772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.347775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.347777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765442.347782:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765442.347823:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.347836:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa4a80 x1876702068005632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.347847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068005632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.347854:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.347857:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.347860:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765442.347865:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765442.348006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.348020:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa5880 x1876702068005760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.348031:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068005760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.348038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.348040:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.348043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068005760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765442.348048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068005888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765442.348068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.348079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa5c00 x1876702068005888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.348090:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068005888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.348097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.348100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.348103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068005888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765442.348109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765442.348127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765442.348138:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765442/real 1789765442] req@ffff936010aa7800 x1876702068006016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765442.348149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068006016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765442.348160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765442.348163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765442.348166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006016:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765442.602962:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765443.626055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765444.650166:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765445.674736:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765446.698646:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765447.465926:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.465936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765447.465941:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.465950:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.465960:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.465966:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765447.466001:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068006144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466049:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466053:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765447.466055:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466059:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466075:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765447.466086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068006272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466095:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466099:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765447.466100:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466104:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466107:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466110:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765447.466123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068006400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765447.466136:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466139:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466142:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466146:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466150:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765447.466169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068006528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466179:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466182:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765447.466183:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466187:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466190:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466193:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466197:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765447.466207:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068006656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765447.466220:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466224:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466227:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466230:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466234:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765447.466243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068006784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765447.466256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765447.466259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765447.466260:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765447.466264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2915 00000100:00080000:1.0:1789765447.466267:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765447.466270:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765447.466274:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765447.466283:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068006912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765447.466373:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068006144:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765447.466431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466476:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef4a80 x1876702068006144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466493:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068006144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466504:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466507:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068006144:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765447.466513:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765447.466534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef5880 x1876702068006272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068006272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466569:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466572:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765447.466580:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765447.466597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466607:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef4380 x1876702068006400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068006400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466643:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765447.466648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765447.466665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466675:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef4000 x1876702068006528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466686:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068006528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466693:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466695:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466699:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765447.466704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765447.466722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef4700 x1876702068006656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068006656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466748:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466753:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068006656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765447.466758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765447.466773:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466782:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef5c00 x1876702068006784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068006784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466837:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466841:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466844:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765447.466850:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765447.466869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765447.466880:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765447/real 1789765447] req@ffff936010ef5180 x1876702068006912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765447.466890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068006912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765447.466898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765447.466901:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765447.466904:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068006912:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765447.722544:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765448.746128:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765449.771028:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765450.794494:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765451.818404:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765452.585940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.585949:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765452.585953:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.585962:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.585972:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.585980:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765452.586013:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068007040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765452.586047:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.586051:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765452.586052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.586056:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586060:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.586064:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765452.586073:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765452.586085:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068007168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765452.586093:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.586096:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765452.586097:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.586100:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586103:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.586107:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765452.586111:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765452.586121:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068007296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765452.586137:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068007040:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765452.586146:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.586150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765452.586151:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.586155:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586159:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.586163:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765452.586168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765452.586181:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068007424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765452.586190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.586193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765452.586195:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789765452.586200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765452.586204:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586209:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.586213:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765452.586217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765452.586241:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061dd500 x1876702068007040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765452.586257:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068007040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586266:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586270:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586274:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068007040:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765452.586281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765452.586301:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765452.586312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061df480 x1876702068007168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765452.586327:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068007168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586334:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586338:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586344:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765452.586366:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765452.586396:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765452.586407:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061df100 x1876702068007296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765452.586416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068007296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586427:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765452.586435:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765452.586454:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765452.586464:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068007552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765452.586465:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061dd880 x1876702068007424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765452.586494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765452.586497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068007424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765452.586499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765452.586502:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.586510:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586516:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765452.586524:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789765452.586527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789765452.586531:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765452.586534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586539:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765452.586547:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068007680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765452.586556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007552:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765452.586572:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765452.586575:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765452.586577:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765452.586581:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2920 00000100:00080000:2.0:1789765452.586584:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789765452.586587:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765452.586590:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765452.586596:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765452.586605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061ddc00 x1876702068007552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765452.586608:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068007808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765452.586633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068007552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586641:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586647:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068007552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765452.586658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068007680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765452.586677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765452.586687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061de300 x1876702068007680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765452.586708:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068007680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586716:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586723:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068007680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765452.586734:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068007808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765452.586753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765452.586764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765452/real 1789765452] req@ffff9360061dc380 x1876702068007808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765452.586773:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068007808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765452.586780:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765452.586783:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765452.586786:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068007808:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765452.842176:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765453.867246:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765454.890100:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765455.914027:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765456.939582:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765457.706010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765457.706021:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765457.706025:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706035:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706046:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765457.706056:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765457.706104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068007936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765457.706127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765457.706130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765457.706131:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706135:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706148:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765457.706154:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706163:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765457.706178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068008064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765457.706179:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068007936:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765457.706195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765457.706198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765457.706200:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706205:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706209:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765457.706212:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765457.706228:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068008192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765457.706237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789765457.706243:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765457.706244:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765457.706248:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706253:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706258:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765457.706262:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706267:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765457.706278:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068008320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765457.706287:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa6300 x1876702068007936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765457.706289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765457.706297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765457.706301:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706308:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706313:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765457.706317:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068007936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765457.706319:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706327:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765457.706334:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765457.706343:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068008448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765457.706347:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068007936:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765457.706364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765457.706368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765457.706370:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706375:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00080000:3.0:1789765457.706379:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765457.706380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008064:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765457.706386:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706392:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765457.706405:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068008576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765457.706406:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765457.706419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765457.706420:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa7b80 x1876702068008064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765457.706427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765457.706431:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765457.706439:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2925 00000100:00100000:0.0:1789765457.706443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068008064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765457.706444:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765457.706452:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765457.706460:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765457.706461:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706468:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706471:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765457.706476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068008704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765457.706481:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765457.706506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765457.706516:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa4000 x1876702068008192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765457.706528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068008192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765457.706536:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706542:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765457.706547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765457.706561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765457.706569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa7800 x1876702068008320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765457.706581:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068008320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765457.706588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706591:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765457.706598:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765457.706611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765457.706620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa5c00 x1876702068008448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765457.706630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068008448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765457.706637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706643:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765457.706660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068008576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765457.706674:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765457.706682:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa5880 x1876702068008576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765457.706693:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068008576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765457.706706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068008576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765457.706716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068008704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765457.706729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765457.706751:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765457/real 1789765457] req@ffff936010aa4a80 x1876702068008704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765457.706762:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068008704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765457.706769:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765457.706772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765457.706775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068008704:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765457.965271:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765458.987261:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765460.010253:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765461.034004:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765462.058106:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765462.825865:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.825874:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765462.825878:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.825887:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.825897:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.825904:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765462.825995:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068008832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826098:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765462.826100:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826105:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826109:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.826113:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826122:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765462.826260:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068008960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765462.826277:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826281:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826285:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765462.826416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068008832:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765462.826541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008960:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765462.826604:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826610:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765462.826634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068009088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826651:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826837:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765462.826840:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826845:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826849:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.826854:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826860:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765462.826873:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068009216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826883:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826885:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765462.826887:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826890:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826894:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.826897:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826901:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765462.826910:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068009344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826918:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826922:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765462.826923:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826926:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826929:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.826932:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826937:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765462.826958:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068009472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765462.826971:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765462.826973:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765462.826975:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765462.826978:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2930 00000100:00080000:3.0:1789765462.826981:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765462.826983:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765462.826987:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765462.826996:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068009600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765462.827189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765462.827220:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765462.827240:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765462.827258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068009472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765462.827275:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068009600:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765463.083542:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765464.107183:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765465.130083:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765465.898107:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765465.898148:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765465.898163:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898201:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898210:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898218:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898226:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898233:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765465.898241:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765465.898321:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa5f80 x1876702068009088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898346:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068009088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898357:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765465.898373:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa7480 x1876702068009216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068009216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765465.898407:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa6680 x1876702068009344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068009344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898432:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898435:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765465.898445:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa5180 x1876702068009472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898457:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068009472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898464:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068009472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765465.898477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa4700 x1876702068009600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898487:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068009600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898498:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898501:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068009600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765465.898507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa4380 x1876702068008832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068008832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068008832:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765465.898540:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765462/real 1789765465] req@ffff936010aa4e00 x1876702068008960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765465.898551:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068008960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765465.898559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765465.898563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765465.898566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068008960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765466.157608:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765467.178066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765467.945974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.945985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765467.945989:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.945999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946009:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946027:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765467.946065:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068009728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765467.946104:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946108:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946111:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946115:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946124:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765467.946136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068009856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946149:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765467.946150:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946154:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946157:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946160:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946165:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765467.946175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068009984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946186:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765467.946188:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946191:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946194:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946197:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946202:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765467.946211:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068010112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946218:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946221:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765467.946223:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946225:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946228:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946232:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946236:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765467.946247:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068010240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946255:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946258:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765467.946260:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946264:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946267:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946271:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946276:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765467.946286:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068010368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765467.946294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765467.946297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765467.946299:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765467.946303:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2935 00000100:00080000:1.0:1789765467.946307:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765467.946310:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765467.946314:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765467.946325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068010496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765467.946413:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068009728:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765467.946470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946505:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef7100 x1876702068009728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946522:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068009728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946532:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068009728:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765467.946546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765467.946568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef4e00 x1876702068009856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068009856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946605:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765467.946611:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765467.946629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946641:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef6d80 x1876702068009984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068009984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068009984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765467.946671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765467.946691:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946702:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef6300 x1876702068010112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068010112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946719:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946722:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765467.946730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765467.946747:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946757:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef7b80 x1876702068010240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946768:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068010240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946781:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765467.946787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068010368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765467.946829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946840:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef7480 x1876702068010368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946851:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068010368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946859:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946862:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946865:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068010368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765467.946870:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068010496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765467.946887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765467.946897:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765467/real 1789765467] req@ffff936010ef6680 x1876702068010496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765467.946907:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068010496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765467.946915:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765467.946918:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765467.946921:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068010496:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765468.202927:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765469.226909:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765470.251201:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765471.276059:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765472.298429:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765473.069984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.069994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765473.069998:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070008:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070019:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070026:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765473.070061:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068010624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765473.070100:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070103:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070111:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070119:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765473.070131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068010752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765473.070146:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070149:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070153:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070156:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070160:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765473.070170:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068010880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070179:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765473.070183:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070186:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070190:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070193:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070197:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765473.070207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068011008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765473.070221:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070224:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070226:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070229:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765473.070240:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068011136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070247:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765473.070251:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070254:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070257:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070261:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070265:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765473.070274:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068011264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765473.070287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765473.070290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765473.070292:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765473.070296:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2940 00000100:00080000:0.0:1789765473.070299:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765473.070302:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765473.070306:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765473.070315:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068011392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765473.070451:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068010624:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765473.070519:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020dac700 x1876702068010624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070570:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068010624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070580:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070584:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070587:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068010624:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765473.070595:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765473.070617:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070629:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020daf100 x1876702068010752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068010752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765473.070660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765473.070679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020dac380 x1876702068010880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068010880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070712:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068010880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765473.070717:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765473.070735:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070746:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020dad180 x1876702068011008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070756:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068011008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070764:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070767:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070769:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765473.070775:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765473.070793:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020dae300 x1876702068011136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070839:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068011136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070845:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765473.070856:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068011264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765473.070875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070886:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020dace00 x1876702068011264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070897:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068011264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070904:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070907:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070910:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068011264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765473.070915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068011392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765473.070932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765473.070943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765473/real 1789765473] req@ffff936020daea00 x1876702068011392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765473.070960:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068011392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765473.070968:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765473.070971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765473.070985:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068011392:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765473.382687:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765474.431059:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765475.437097:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765476.458397:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765477.485765:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765478.192771:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.192781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765478.192785:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.192795:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.192845:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.192852:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765478.192888:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068011520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.192920:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.192924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765478.192926:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.192929:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.192932:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.192944:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.192953:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765478.192963:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068011648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.192972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.192975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765478.192977:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.192980:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.192983:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.192986:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.192990:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765478.193000:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068011776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.193009:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.193012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765478.193013:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.193017:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.193020:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.193023:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.193028:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765478.193037:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068011904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.193046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.193050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765478.193051:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.193054:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.193057:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.193061:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.193065:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765478.193073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068012032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.193082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.193085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765478.193087:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.193090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.193093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.193096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.193100:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765478.193108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068012160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765478.193117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765478.193119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765478.193121:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765478.193124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2946 00000100:00080000:2.0:1789765478.193126:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765478.193129:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765478.193132:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765478.193140:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068012288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765478.194680:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068011520:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765478.194742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.194779:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061dc700 x1876702068011520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.194796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068011520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.194835:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.194839:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.194842:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068011520:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765478.194849:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765478.194871:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.194882:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061ddf80 x1876702068011648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.194893:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068011648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.194908:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.194911:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.194914:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765478.194920:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765478.194947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.194958:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061dc380 x1876702068011776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.194968:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068011776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.194976:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.194979:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.194982:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765478.194988:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765478.195006:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.195016:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061de300 x1876702068011904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.195026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068011904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.195033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.195037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.195041:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068011904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765478.195047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765478.195065:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.195075:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061ddc00 x1876702068012032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.195085:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068012032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.195093:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.195096:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.195099:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765478.195104:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068012160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765478.195119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.195127:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061dd880 x1876702068012160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.195138:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068012160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.195145:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.195148:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.195151:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068012160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765478.195156:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068012288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765478.195186:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765478.195196:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765478/real 1789765478] req@ffff9360061df100 x1876702068012288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765478.195206:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068012288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765478.195214:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765478.195217:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765478.195220:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068012288:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765478.506559:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765479.529986:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765480.554017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765481.578095:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765482.601944:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765483.306927:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.306936:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765483.306940:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.306948:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.306959:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.306965:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765483.306999:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068012416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307037:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307041:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765483.307043:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307047:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307051:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307054:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765483.307073:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068012544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307085:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765483.307086:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307093:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307096:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307100:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765483.307109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068012672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765483.307122:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307125:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307128:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307132:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765483.307147:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068012800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765483.307160:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307163:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307166:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307169:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307172:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765483.307182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068012928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765483.307194:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307197:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307204:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307209:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765483.307219:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068013056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765483.307227:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765483.307230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765483.307231:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765483.307235:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2951 00000100:00080000:3.0:1789765483.307238:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765483.307240:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765483.307244:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765483.307252:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068013184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765483.307489:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068012416:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765483.307544:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.307981:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa7100 x1876702068012416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.307997:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068012416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308022:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068012416:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765483.308034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765483.308058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308071:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa5500 x1876702068012544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068012544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765483.308109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765483.308129:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa6a00 x1876702068012672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068012672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308166:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765483.308172:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765483.308192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa4e00 x1876702068012800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308214:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068012800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308227:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308230:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308233:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765483.308238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765483.308258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308269:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa4380 x1876702068012928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068012928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308287:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308293:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068012928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765483.308299:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765483.308318:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa4700 x1876702068013056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308341:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068013056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308348:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308351:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308354:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765483.308360:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765483.308385:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765483.308397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765483/real 1789765483] req@ffff936010aa5180 x1876702068013184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765483.308408:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068013184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765483.308416:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765483.308423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765483.308426:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013184:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765483.627497:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765484.651985:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765485.674045:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765486.698203:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765487.722157:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765488.426001:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765488.426016:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426027:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426037:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426045:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765488.426083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068013312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426118:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765488.426120:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426141:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765488.426152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068013440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426162:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426165:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765488.426167:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426171:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426177:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426182:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765488.426191:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068013568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426200:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426203:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765488.426205:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426208:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426211:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426215:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765488.426228:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068013696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765488.426242:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426246:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426249:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426252:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426257:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765488.426266:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068013824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765488.426280:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426283:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426286:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426290:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426293:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765488.426302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068013952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765488.426310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765488.426313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765488.426314:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765488.426318:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2956 00000100:00080000:2.0:1789765488.426321:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765488.426324:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765488.426329:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765488.426340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068014080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765488.426422:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068013312:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765488.426490:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426527:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061df480 x1876702068013312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068013312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426553:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068013312:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765488.426567:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765488.426588:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426599:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061dd500 x1876702068013440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426609:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068013440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765488.426627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765488.426645:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061dc000 x1876702068013568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426665:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068013568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426675:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426678:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765488.426683:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765488.426701:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426711:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061dce00 x1876702068013696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068013696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426737:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426740:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765488.426749:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765488.426767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426778:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061dd180 x1876702068013824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426790:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068013824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426834:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426837:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426841:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068013824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765488.426847:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765488.426878:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061dca80 x1876702068013952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426899:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068013952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426911:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426914:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426926:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068013952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765488.426932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765488.426951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765488.426962:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765488/real 1789765488] req@ffff9360061de680 x1876702068014080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765488.426972:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068014080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765488.426980:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765488.426983:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765488.426986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014080:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765488.746269:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765489.773864:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765490.794475:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765491.819396:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765492.842085:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765493.546189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765493.546199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765493.546203:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546214:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546224:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546231:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765493.546267:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068014208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765493.546299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765493.546303:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765493.546304:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546308:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546312:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546315:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546323:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765493.546335:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068014336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765493.546347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068014208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765493.546362:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765493.546366:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765493.546368:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546373:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546377:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546381:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546386:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765493.546407:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068014464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789765493.546417:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765493.546427:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765493.546430:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765493.546432:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546437:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546441:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546444:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546449:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765493.546462:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068014592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765493.546470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765493.546473:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020daf800 x1876702068014208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765493.546474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765493.546477:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546485:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546490:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546495:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546500:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765493.546502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068014208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765493.546511:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068014720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765493.546515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.546525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765493.546527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789765493.546532:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068014208:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765493.546533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765493.546538:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546545:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546550:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546554:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546559:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765493.546560:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765493.546573:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068014848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765493.546585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789765493.546590:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765493.546591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765493.546594:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765493.546599:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2961 00000100:00080000:0.0:1789765493.546602:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765493.546605:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765493.546608:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765493.546618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068014976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765493.546720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020dadc00 x1876702068014336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.546733:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068014336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.546740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.546743:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.546746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765493.546751:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765493.546770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765493.546780:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020dadf80 x1876702068014464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.546789:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068014464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.546796:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.546821:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.546837:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765493.546842:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765493.546859:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765493.546869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020daea00 x1876702068014592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.546879:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068014592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.546896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.546899:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.546902:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765493.546908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765493.546926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765493.546948:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020dace00 x1876702068014720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.546958:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068014720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.546965:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.546968:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.546970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068014720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765493.546986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765493.547004:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765493.547015:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020dae300 x1876702068014848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.547025:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068014848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.547032:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.547035:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.547038:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765493.547042:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765493.547058:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765493.547066:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765493/real 1789765493] req@ffff936020dad180 x1876702068014976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765493.547075:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068014976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765493.547081:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765493.547084:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765493.547086:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068014976:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765493.870982:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765494.890373:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765495.914202:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765496.938003:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765497.962048:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765498.666018:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765498.666033:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666043:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666053:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666061:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765498.666096:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068015104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666129:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765498.666135:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666139:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666143:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666146:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765498.666166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068015232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765498.666179:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666182:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666184:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666187:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666191:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765498.666201:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068015360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765498.666213:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666216:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666219:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666222:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666226:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765498.666235:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068015488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765498.666248:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666251:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666253:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666257:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666261:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765498.666270:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068015616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765498.666282:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666284:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666287:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666290:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666294:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765498.666303:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068015744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765498.666311:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765498.666313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765498.666314:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765498.666317:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2966 00000100:00080000:2.0:1789765498.666320:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765498.666322:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765498.666326:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765498.666334:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068015872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765498.666497:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068015104:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765498.666548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.666583:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061dfb80 x1876702068015104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.666597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068015104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.666606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.666610:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.666615:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068015104:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765498.666622:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765498.666642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.666653:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061df800 x1876702068015232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.666664:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068015232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.666671:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.666674:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.666676:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765498.666681:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765498.666991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.667005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061de680 x1876702068015360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.667015:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068015360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.667023:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.667026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.667029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765498.667034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765498.667054:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.667065:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061dca80 x1876702068015488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.667076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068015488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.667083:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.667086:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.667089:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765498.667094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765498.667112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.667123:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061dd180 x1876702068015616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.667133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068015616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.667139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.667142:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.667144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068015616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765498.667149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068015744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765498.667167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.667177:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061dce00 x1876702068015744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.667186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068015744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.667194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.667197:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.667199:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068015744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765498.667204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068015872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765498.667219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765498.667228:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765498/real 1789765498] req@ffff9360061dc000 x1876702068015872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765498.667238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068015872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765498.667244:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765498.667247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765498.667249:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068015872:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765498.990834:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765500.010491:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765501.034101:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765502.066163:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765503.082217:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765503.786489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786503:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765503.786508:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786521:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786535:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786544:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765503.786595:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068016000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786639:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786643:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765503.786645:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786650:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786654:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786658:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786670:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765503.786684:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068016128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786694:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765503.786699:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786703:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786707:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786711:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786715:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765503.786726:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068016256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786739:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765503.786741:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786745:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786760:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786764:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786770:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765503.786781:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068016384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786789:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786792:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765503.786793:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786824:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786828:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786832:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786836:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765503.786849:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068016512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786859:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786862:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765503.786864:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786867:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786871:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786875:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786879:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765503.786890:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068016640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765503.786911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765503.786915:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765503.786917:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765503.786921:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2971 00000100:00080000:1.0:1789765503.786924:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765503.786928:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765503.786932:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765503.786942:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068016768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765503.787371:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068016000:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765503.787452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.787501:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef7800 x1876702068016000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.787521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068016000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.787539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.787543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.787547:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068016000:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765503.787555:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765503.787579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.787592:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef5500 x1876702068016128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.787604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068016128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765503.788022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765503.788048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.788061:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef5f80 x1876702068016256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.788072:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068016256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788081:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788084:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788088:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765503.788095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765503.788118:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.788129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef6a00 x1876702068016384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.788141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068016384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788155:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765503.788162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765503.788182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.788353:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef6680 x1876702068016512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.788366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068016512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788377:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788380:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068016512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765503.788386:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068016640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765503.788408:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.788419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef7480 x1876702068016640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.788432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068016640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788440:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068016640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765503.788453:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068016768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765503.788473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765503.788484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765503/real 1789765503] req@ffff936010ef7b80 x1876702068016768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765503.788496:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068016768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765503.788504:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765503.788507:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765503.788510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068016768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765504.106555:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765505.130670:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765506.154078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765507.178248:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765508.203599:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765508.905898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.905908:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765508.905912:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.905923:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.905934:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.905941:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765508.905979:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068016896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765508.906007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765508.906013:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906016:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906020:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906024:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906032:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765508.906043:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068017024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765508.906057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765508.906062:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906065:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906068:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906072:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906076:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765508.906087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068017152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765508.906096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765508.906100:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906103:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906106:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906110:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906114:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765508.906123:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068017280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765508.906131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765508.906136:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906139:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906142:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906145:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906149:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765508.906157:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068017408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765508.906164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765508.906169:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906172:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906175:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906178:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765508.906190:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068017536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765508.906281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068016896:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765508.906334:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.906367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef6300 x1876702068016896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.906381:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068016896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.906391:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.906394:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.906398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068016896:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765508.906403:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765508.906420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.906431:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef6d80 x1876702068017024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.906440:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068017024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.906447:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.906450:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.906453:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765508.906458:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765508.906472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.906481:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef4e00 x1876702068017152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.906537:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068017152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.906544:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.906547:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.906550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765508.906556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765508.906574:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.906583:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef7100 x1876702068017280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.906597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068017280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.906604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.906607:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.906609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765508.906614:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765508.906629:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.906638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef5180 x1876702068017408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.906648:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068017408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.906655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.906657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.906660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765508.906915:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068017536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765508.906939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765508.906963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765508.906967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765508.906969:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765508.906973:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2976 00000100:00080000:1.0:1789765508.906978:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765508.906982:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765508.906987:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765508.907000:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068017664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765508.907043:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef5c00 x1876702068017536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.907056:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068017536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.907064:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.907067:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.907070:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068017536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765508.907083:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068017664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765508.907104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765508.907115:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765508/real 1789765508] req@ffff936010ef4700 x1876702068017664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765508.907126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068017664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765508.907132:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765508.907135:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765508.907137:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068017664:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765509.227762:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765510.250117:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765511.275542:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765512.298294:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765513.322181:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765514.025976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.025985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765514.025989:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.025999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026015:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765514.026048:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068017792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026075:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765514.026080:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026084:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026086:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026090:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026099:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765514.026109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068017920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765514.026122:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026125:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026130:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026134:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765514.026142:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068018048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026149:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765514.026154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026157:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026159:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026162:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026166:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765514.026173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068018176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026180:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765514.026184:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026187:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026190:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026192:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026196:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765514.026215:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068018304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026224:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026227:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765514.026229:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026239:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026242:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026246:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026250:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765514.026259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068018432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765514.026272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765514.026275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765514.026276:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765514.026279:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2981 00000100:00080000:2.0:1789765514.026282:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765514.026285:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765514.026288:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765514.026295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068018560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765514.026438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068017792:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765514.026504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026540:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061dd500 x1876702068017792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026555:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068017792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026570:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026573:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068017792:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765514.026580:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765514.026601:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026613:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061df480 x1876702068017920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068017920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068017920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765514.026642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765514.026660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026669:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061df100 x1876702068018048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068018048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026705:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765514.026710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765514.026728:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026738:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061dd880 x1876702068018176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068018176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026755:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026757:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765514.026766:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765514.026783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061ddc00 x1876702068018304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068018304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026839:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026842:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026845:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765514.026850:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068018432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765514.026870:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026881:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061de300 x1876702068018432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026892:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068018432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026900:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026903:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068018432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765514.026908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068018560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765514.026923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765514.026932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765514/real 1789765514] req@ffff9360061dc380 x1876702068018560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765514.026942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068018560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765514.026948:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765514.026950:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765514.026953:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068018560:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765514.346569:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765515.371249:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765516.394652:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765517.418660:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765518.443073:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765519.146176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765519.146194:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146206:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146220:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146228:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765519.146279:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068018688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765519.146323:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146327:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146331:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146335:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765519.146361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068018816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146370:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146373:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765519.146375:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146379:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146382:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146386:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146391:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765519.146403:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068018944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146417:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765519.146418:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146422:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146426:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146429:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146434:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765519.146445:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068019072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146453:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765519.146459:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146462:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146465:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146468:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146472:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765519.146482:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068019200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765519.146496:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146499:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146502:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146506:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146510:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765519.146520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068019328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765519.146528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765519.146531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765519.146533:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765519.146537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2986 00000100:00080000:2.0:1789765519.146540:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765519.146543:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765519.146548:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765519.146558:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068019456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765519.148322:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068018688:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765519.148418:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.148488:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061ddf80 x1876702068018688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.148510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068018688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.148521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.148526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.148530:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068018688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765519.148539:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765519.148565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.148577:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061dc700 x1876702068018816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.148590:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068018816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.148599:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.148602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.148605:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765519.148612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765519.148648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.148660:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061dc380 x1876702068018944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.148671:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068018944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.148680:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.148683:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.148697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068018944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765519.148705:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765519.148726:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.148738:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061de300 x1876702068019072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.148751:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068019072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.148758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.148762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.148765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765519.148773:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765519.148794:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.149082:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061ddc00 x1876702068019200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.149095:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068019200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.149102:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.149105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.149108:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765519.149115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068019328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765519.149137:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.149148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061dd880 x1876702068019328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.149159:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068019328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.149166:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.149169:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.149172:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068019328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765519.149178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068019456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765519.149200:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765519.149209:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765519/real 1789765519] req@ffff9360061df100 x1876702068019456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765519.149218:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068019456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765519.149225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765519.149228:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765519.149230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068019456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765519.466224:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765520.490082:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765521.514682:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765522.537992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765523.562241:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765524.266065:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.266074:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765524.266078:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.266087:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.266097:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.266105:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765524.266200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068019584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.266328:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068019584:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765524.266399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765524.266477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.266481:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765524.266483:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.266487:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.266492:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.266495:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.266505:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765524.266518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068019712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.266579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061df480 x1876702068019584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.266610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068019584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.266620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.266624:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.266628:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068019584:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765524.266651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765524.266675:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765524.266689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061dd500 x1876702068019712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.266701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068019712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.266709:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.266712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.266715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019712:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765524.266880:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.266893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765524.266895:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.266900:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.266905:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.266910:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.266916:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765524.267028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068019840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.267070:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765524.267094:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765524.267121:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.267125:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765524.267127:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.267131:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.267135:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.267140:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.267146:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765524.267157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068019968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.267294:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061dc000 x1876702068019840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.267306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068019840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.267313:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.267316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.267319:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765524.267331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765524.267350:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765524.267359:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061dce00 x1876702068019968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.267430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068019968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.267439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.267443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.267446:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068019968:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765524.267529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.267533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765524.267534:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.267538:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.267542:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.267546:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.267551:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765524.267563:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068020096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.267611:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765524.267634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765524.267654:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.267658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765524.267660:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.267664:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.267668:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.267672:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.267677:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765524.267688:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068020224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.267723:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061dd180 x1876702068020096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.267734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068020096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.267741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.267743:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.267746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765524.267760:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068020224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765524.267779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765524.267790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061dca80 x1876702068020224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.268094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068020224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.268103:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.268106:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.268109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068020224:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765524.268851:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765524.268855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765524.268858:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765524.268862:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2992 00000100:00080000:2.0:1789765524.268867:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765524.268872:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765524.268878:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765524.268900:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068020352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765524.268996:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068020352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765524.269030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765524.269048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765524/real 1789765524] req@ffff9360061de680 x1876702068020352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765524.269061:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068020352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765524.269070:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765524.269073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765524.269077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068020352:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765524.587485:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765525.609956:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765526.634061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765527.658746:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765528.684667:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765529.392280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392289:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765529.392293:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392303:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.392313:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.392320:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765529.392358:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068020480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.392382:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765529.392388:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392392:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.392395:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.392399:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.392407:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765529.392418:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068020608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.392428:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392431:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765529.392433:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392436:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.392439:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.392441:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.392445:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765529.392453:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068020736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.392462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765529.392467:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392470:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.392473:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.392477:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.392481:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765529.392488:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068020864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.392497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765529.392501:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392504:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.392507:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.392510:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.392513:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765529.392520:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068020992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.392527:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.392530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765529.392531:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.392534:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00100000:1.0:1789765529.392669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068020480:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765529.392757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765529.392780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765529.392829:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765529.392852:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020992:192.168.203.137@tcp:8:kworker.0 00000100:00080000:0.0:1789765529.393064:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.393070:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.393076:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765529.393099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068021120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765529.393121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765529.393125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765529.393126:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765529.393130:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 2997 00000100:00080000:0.0:1789765529.393133:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765529.393137:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765529.393141:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765529.393150:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068021248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765529.393285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068021120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765529.393313:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068021248:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765529.706118:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765530.731957:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765531.754527:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765532.458884:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765532.458925:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765532.458941:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765532.460274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020dac380 x1876702068020480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765554 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.460300:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068020480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765554 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.460312:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.460316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.460320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068020480:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765532.461093:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765532.461121:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765532.461129:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765532.461136:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765532.461142:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765532.461148:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789765532.462838:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020dafb80 x1876702068021120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.462866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068021120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.462883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.462887:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.462892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068021120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765532.462899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020daf480 x1876702068021248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.462911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068021248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.462919:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.462922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.462925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068021248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765532.462931:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020daf100 x1876702068020608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.462942:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068020608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.462949:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.462953:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.462956:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765532.462963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020dac700 x1876702068020736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.462974:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068020736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.462981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.462984:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.462988:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765532.462994:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020daed80 x1876702068020864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.463003:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068020864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.463025:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.463028:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.463031:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765532.463037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765529/real 1789765532] req@ffff936020dae680 x1876702068020992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765532.463049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068020992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765532.463056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765532.463059:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765532.463062:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068020992:192.168.203.137@tcp:8:kworker.0 00010000:00010000:3.0:1789765532.779390:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765533.808083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765534.517782:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.517791:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765534.517795:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.517840:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.517850:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.517858:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765534.517895:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068021376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.517928:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.517931:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765534.517933:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.517936:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.517940:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.517944:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.517952:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765534.517963:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068021504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.517972:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.517975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765534.517976:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.517980:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.517983:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.517986:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.517990:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765534.518001:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068021632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.518010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.518013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765534.518014:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.518017:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.518021:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.518024:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.518027:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765534.518036:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068021760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.518044:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.518047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765534.518049:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.518052:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.518055:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.518058:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.518062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765534.518070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068021888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.518078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.518081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765534.518083:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.518086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00080000:2.0:1789765534.518089:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.518093:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.518097:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765534.518106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068022016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765534.518114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765534.518116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765534.518118:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765534.518121:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3002 00000100:00100000:1.0:1789765534.518324:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068021376:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765534.518394:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518430:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061df800 x1876702068021376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518446:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068021376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068021376:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765534.518471:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765534.518504:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518516:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061dfb80 x1876702068021504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518547:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068021504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518554:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765534.518565:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765534.518584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061de680 x1876702068021632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068021632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518618:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518621:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765534.518636:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765534.518655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518665:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061dca80 x1876702068021760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518674:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068021760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518684:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765534.518691:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765534.518706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061dd180 x1876702068021888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518723:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068021888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518730:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518732:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518735:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068021888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765534.518740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765534.518756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.518766:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061dce00 x1876702068022016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.518775:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068022016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.518782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.518785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.518788:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022016:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765534.519350:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765534.519358:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765534.519364:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765534.519383:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068022144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765534.520644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765534.520697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765534.520720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765534/real 1789765534] req@ffff9360061dc000 x1876702068022144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765534.520747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068022144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765534.520757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765534.520761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765534.520764:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022144:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765534.825948:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765535.851892:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765536.876376:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765537.899963:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765538.928204:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765539.626035:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765539.626056:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626070:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626084:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626092:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765539.626136:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068022272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765539.626178:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626182:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626186:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626190:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765539.626213:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068022400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626226:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765539.626228:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626235:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626239:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626244:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765539.626252:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068022528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626269:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765539.626271:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626275:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626278:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626281:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626287:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765539.626295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068022656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626306:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765539.626307:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626310:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626313:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626316:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626320:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765539.626329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068022784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626336:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765539.626340:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626342:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626345:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626349:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626352:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765539.626361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068022912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765539.626369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765539.626371:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765539.626373:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765539.626376:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3007 00000100:00080000:1.0:1789765539.626380:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765539.626383:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765539.626386:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765539.626395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068023040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765539.626554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068022272:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765539.626617:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.626657:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef4000 x1876702068022272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.626679:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068022272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.626689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.626693:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.626697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068022272:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765539.626703:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765539.626726:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.626738:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef4380 x1876702068022400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.626750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068022400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.626758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.626761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.626763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765539.626768:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765539.626788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.626827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef5880 x1876702068022528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.626838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068022528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.626845:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.626847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.626850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765539.626855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765539.627692:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.627730:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef4a80 x1876702068022656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.627748:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068022656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.627764:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.627767:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.627770:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765539.627778:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765539.627844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.627856:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef4700 x1876702068022784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.627865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068022784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.628857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.629350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.629358:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068022784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765539.629370:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765539.629431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.629464:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef5c00 x1876702068022912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.629479:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068022912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.629498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.629501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.629505:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068022912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765539.629512:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765539.629533:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765539.629545:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765539/real 1789765539] req@ffff936010ef5180 x1876702068023040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765539.629557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068023040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765539.629565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765539.629568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765539.629571:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023040:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765540.106280:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765541.038649:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765542.059324:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765543.165499:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765544.110053:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765544.745874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.745881:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765544.745885:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.745893:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.745902:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.745908:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765544.745947:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068023168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.745981:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.745985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765544.745987:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.745991:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.745995:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.745999:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746008:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765544.746026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068023296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.746036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.746039:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765544.746041:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.746045:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.746048:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.746051:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746055:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765544.746067:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068023424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.746086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.746090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765544.746092:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.746096:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.746099:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.746103:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746108:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765544.746118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068023552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.746127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.746130:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765544.746131:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.746134:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.746137:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.746140:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746144:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765544.746152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068023680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.746160:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.746163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765544.746165:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765544.746168:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.746171:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.746174:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746179:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765544.746188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068023808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765544.746198:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765544.746201:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765544.746203:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765544.746207:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068023168:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765544.746209:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3012 00000100:00080000:1.0:1789765544.746216:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765544.746220:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765544.746225:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765544.746234:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068023936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765544.746280:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746322:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef7100 x1876702068023168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746342:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068023168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746352:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746360:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068023168:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765544.746367:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765544.746404:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746416:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef4e00 x1876702068023296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746428:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068023296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746436:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746440:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746443:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765544.746466:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765544.746485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746501:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef6d80 x1876702068023424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746522:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068023424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746539:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746542:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746545:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765544.746551:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765544.746569:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746579:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef6300 x1876702068023552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068023552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746604:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746611:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765544.746616:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765544.746634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef7b80 x1876702068023680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746653:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068023680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746659:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746672:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746675:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068023680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765544.746701:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765544.746718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746727:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef7480 x1876702068023808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746737:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068023808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746748:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746750:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746753:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765544.746758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765544.746772:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765544.746781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765544/real 1789765544] req@ffff936010ef6680 x1876702068023936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765544.746791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068023936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765544.746826:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765544.746830:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765544.746833:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068023936:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765545.136672:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765546.154825:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765547.178109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765548.207173:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765549.226049:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765549.866359:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.866370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765549.866374:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.866383:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.866394:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.866402:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765549.866448:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068024064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765549.866531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068024064:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765549.866594:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765549.866624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.866628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765549.866630:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.866634:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.866639:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.866643:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.866652:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765549.866665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068024192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765549.871667:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061dd500 x1876702068024064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765549.871696:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068024064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765549.871707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765549.871713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765549.871717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068024064:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765549.871746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765549.871817:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765549.871834:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061df480 x1876702068024192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765549.871847:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068024192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765549.871855:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765549.871858:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765549.871861:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024192:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765549.873117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.873124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765549.873130:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.873137:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.873144:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.873151:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.873162:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765549.873191:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068024320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765549.873241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.873245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765549.873246:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.873250:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.873254:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.873257:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.873262:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765549.873272:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068024448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765549.873280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.873282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765549.873284:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.873286:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.873289:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.873292:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.873295:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765549.873302:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068024576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765549.873309:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.873312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765549.873313:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.873316:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.873318:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.873321:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.873325:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765549.873332:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068024704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765549.873346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765549.873348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765549.873350:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765549.873353:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3017 00000100:00080000:2.0:1789765549.873356:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765549.873359:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765549.873362:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765549.873371:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068024832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765549.873537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765549.873582:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765549.873601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061df100 x1876702068024320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765549.873616:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068024320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765549.873625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765549.873628:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765549.873631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765549.873638:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765549.873660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765549.873672:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061dd880 x1876702068024448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765549.873683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068024448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765549.873691:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765549.873694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765549.873697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765549.873703:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765549.873722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765549.873733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061ddc00 x1876702068024576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765549.873743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068024576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765549.873750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765549.873752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765549.873755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068024576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765549.873760:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068024704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765549.873779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765549.873790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061de300 x1876702068024704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765549.873826:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068024704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765549.873835:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765549.873839:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765549.873842:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068024704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765549.873848:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068024832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765549.873866:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765549.873876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765549/real 1789765549] req@ffff9360061dc380 x1876702068024832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765549.873887:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068024832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765549.873894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765549.873897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765549.873901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068024832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765550.249998:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765551.274030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765552.299690:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765553.322130:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765554.347626:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765554.985994:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.986009:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765554.986015:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.986029:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.986043:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.986052:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765554.986108:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068024960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765554.986210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068024960:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765554.986289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765554.986326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.986330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765554.986332:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.986346:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.986350:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.986355:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.986367:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765554.986391:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068025088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765554.987836:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061dc700 x1876702068024960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.987873:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068024960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.987886:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.987891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.987895:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068024960:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765554.988071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765554.988128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.988146:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061ddf80 x1876702068025088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.988160:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068025088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.988169:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.988172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.988176:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025088:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765554.989173:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.989183:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765554.989187:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.989196:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.989207:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.989215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.989228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765554.989270:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068025216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765554.989285:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.989289:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765554.989291:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.989295:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.989298:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.989302:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.989307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765554.989317:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068025344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765554.989326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.989329:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765554.989332:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.989335:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.989338:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.989341:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.989345:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765554.989353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068025472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765554.989363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.989366:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765554.989368:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.989389:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00080000:2.0:1789765554.989392:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.989396:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.989400:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765554.989410:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068025600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765554.989419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765554.989430:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765554.989432:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765554.989436:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3022 00000100:00100000:3.0:1789765554.989554:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765554.989620:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.989646:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061ddf80 x1876702068025216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.989664:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068025216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.989681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.989685:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.989688:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765554.989696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765554.989717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.989729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061dc700 x1876702068025344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.989742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068025344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.989751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.989754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.989758:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765554.989764:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765554.989784:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.989795:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061dc380 x1876702068025472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.989837:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068025472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.989849:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.989853:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.989858:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765554.989865:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068025600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765554.989888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.989897:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061de300 x1876702068025600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.989921:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068025600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.989929:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789765554.990289:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765554.990295:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765554.990303:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765554.990320:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068025728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765554.990698:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.990708:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068025600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765554.990748:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068025728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765554.990780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765554.990826:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765554/real 1789765554] req@ffff9360061ddc00 x1876702068025728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765554.990843:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068025728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765554.990852:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765554.990856:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765554.990859:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068025728:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765555.370040:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765556.399263:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765557.420825:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765558.446016:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765559.470580:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765560.108685:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.108699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765560.108704:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.108716:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.108730:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.108738:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765560.108828:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068025856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.108861:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.108865:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765560.108867:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.108871:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.108875:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.108879:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.108891:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765560.108902:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068025984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.108917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.108920:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765560.108922:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.108926:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.108929:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.108932:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.108935:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765560.108944:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068026112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.108952:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.108956:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765560.108957:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.108961:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.108964:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.108968:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.108972:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765560.108981:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068026240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.108990:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.108992:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765560.108994:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.108997:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.109000:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.109004:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.109008:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765560.109018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068026368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.109026:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.109029:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765560.109031:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.109034:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.109036:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.109039:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.109043:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765560.109050:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068026496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765560.109058:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765560.109061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765560.109063:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765560.109066:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3027 00000100:00080000:1.0:1789765560.109069:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765560.109073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765560.109076:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765560.109083:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068026624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765560.109216:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068025856:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765560.109288:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109355:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef6a00 x1876702068025856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109373:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068025856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109385:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068025856:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765560.109404:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765560.109428:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109441:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef5f80 x1876702068025984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068025984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068025984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765560.109473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765560.109489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109498:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef5500 x1876702068026112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109510:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068026112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765560.109530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765560.109546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109555:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef7800 x1876702068026240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109567:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068026240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765560.109585:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765560.109601:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef6680 x1876702068026368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068026368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765560.109644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068026496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765560.109661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109669:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef7480 x1876702068026496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068026496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109687:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068026496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765560.109698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068026624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765560.109714:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765560.109724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765560/real 1789765560] req@ffff936010ef7b80 x1876702068026624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765560.109734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068026624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765560.109742:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765560.109744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765560.109748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068026624:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765560.490641:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765561.514203:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765562.538759:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765563.561981:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765564.586330:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765565.229528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.229538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765565.229543:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765565.229553:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.229564:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.229582:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765565.229618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068026752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765565.229685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068026752:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765565.229732:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765565.229760:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.229764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765565.229766:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765565.229770:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.229774:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.229778:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765565.229786:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765565.229822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068026880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765565.229884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.229888:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765565.229890:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765565.229894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.229898:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.229903:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765565.229906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020dac000 x1876702068026752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789765565.229908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765565.229919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068027008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765565.229924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068026752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765565.229929:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.229932:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765565.229934:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765565.229934:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765565.229938:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:3.0:1789765565.229938:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765565.229941:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789765565.229941:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068026752:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765565.229945:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765565.229948:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765565.229957:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068027136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765565.229962:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765565.229965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.229968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765565.229969:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765565.229972:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.229975:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.229978:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765565.229982:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789765565.229982:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765565.229993:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020dad880 x1876702068026880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765565.230009:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068027264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765565.230011:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068026880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765565.230018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765565.230019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765565.230021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765565.230022:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789765565.230023:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765565.230025:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068026880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765565.230026:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.230029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.230033:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230033:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027008:192.168.203.137@tcp:38:kworker.0 00000008:00100000:0.0:1789765565.230036:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765565.230044:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068027392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789765565.230055:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765565.230059:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765565.230062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765565.230064:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765565.230066:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020dad500 x1876702068027008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765565.230067:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3033 00000100:00080000:0.0:1789765565.230070:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765565.230073:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068027008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789765565.230077:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789765565.230084:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789765565.230085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068027520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765565.230087:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765565.230090:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765565.230103:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765565.230122:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230133:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020dae680 x1876702068027136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765565.230143:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068027136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765565.230150:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765565.230153:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765565.230156:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765565.230162:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765565.230180:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230190:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020daed80 x1876702068027264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765565.230202:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068027264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765565.230327:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765565.230331:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765565.230335:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765565.230341:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068027392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765565.230364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230375:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020dac700 x1876702068027392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765565.230387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068027392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765565.230394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765565.230397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765565.230399:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068027392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765565.230405:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068027520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765565.230423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765565.230433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765565/real 1789765565] req@ffff936020daf100 x1876702068027520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765565.230444:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068027520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765565.230456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765565.230459:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765565.230462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068027520:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765565.614391:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765566.633996:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765567.659679:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765568.687120:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765569.705993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765570.347038:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347048:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765570.347052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347062:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347073:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347080:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765570.347117:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068027648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.347147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347151:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765570.347153:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347157:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347161:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347164:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.347174:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765570.347187:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068027776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.347197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347200:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765570.347202:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347209:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347213:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.347217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765570.347228:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068027904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.347237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347240:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765570.347242:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347257:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347270:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347274:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.347278:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765570.347289:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068028032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.347297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765570.347302:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347305:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347308:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347311:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.347316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765570.347325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068028160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.347344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.347348:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765570.347349:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.347353:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.347356:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.347360:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.347364:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765570.347530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068027648:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765570.347591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.347647:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061dd880 x1876702068027648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.347664:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068027648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.347675:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.347679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.347683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068027648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765570.347690:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765570.347713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.347726:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061df100 x1876702068027776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.347738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068027776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.347753:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.347756:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.347759:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765570.347765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765570.347785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.347796:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061df480 x1876702068027904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.347845:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068027904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.347853:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.347857:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.347860:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068027904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765570.347866:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765570.347887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.347898:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061dd500 x1876702068028032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.347908:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068028032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.347916:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.347919:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.347922:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765570.347928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765570.347962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.347973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061dc000 x1876702068028160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.347985:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068028160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.347993:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.347996:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.347999:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789765570.348307:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068028288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765570.348330:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765570.348334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765570.348336:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765570.348340:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3038 00000100:00080000:2.0:1789765570.348345:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765570.348349:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765570.348368:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765570.348460:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068028416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765570.349346:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068028288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765570.349395:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.349414:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061dce00 x1876702068028288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.349430:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068028288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.349439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.349443:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.349447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068028288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765570.349453:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068028416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765570.349485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765570.349498:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765570/real 1789765570] req@ffff9360061dd180 x1876702068028416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765570.349678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068028416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765570.349687:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765570.349691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765570.349695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068028416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765570.730134:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765571.757113:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765572.778685:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765573.802380:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765574.826525:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765575.468268:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765575.468291:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468302:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468313:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468320:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765575.468358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068028544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765575.468397:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468400:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468404:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468407:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468416:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765575.468427:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068028672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468437:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765575.468441:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468445:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468448:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468452:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468456:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765575.468466:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068028800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468478:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765575.468480:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468483:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468486:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468489:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468493:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765575.468503:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068028928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765575.468515:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468527:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468531:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468535:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468539:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765575.468549:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068029056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765575.468563:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468566:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468569:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468572:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468577:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765575.468586:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068029184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765575.468594:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765575.468596:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765575.468598:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765575.468602:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3043 00000100:00080000:1.0:1789765575.468605:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765575.468609:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765575.468613:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765575.468622:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068029312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765575.470083:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068028544:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765575.470152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.470209:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef6300 x1876702068028544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.470237:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068028544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.470247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.470251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.470254:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068028544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765575.470262:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765575.470285:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.470297:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef6d80 x1876702068028672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.470313:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068028672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.470321:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.470324:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.470327:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765575.470333:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765575.470354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.470365:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef4e00 x1876702068028800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.470376:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068028800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.470384:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.470399:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.470403:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765575.470408:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765575.470648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.470663:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef7100 x1876702068028928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.470683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068028928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.470695:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.470699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.470701:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068028928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765575.470708:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765575.470729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.470739:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef5180 x1876702068029056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.470750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068029056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.470757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.470760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.470762:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765575.471095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068029184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765575.471120:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.471132:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef5c00 x1876702068029184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.471143:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068029184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.471149:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.471152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.471156:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068029184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765575.471161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068029312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765575.471179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765575.471190:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765575/real 1789765575] req@ffff936010ef4700 x1876702068029312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765575.471201:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068029312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765575.471208:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765575.471211:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765575.471214:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068029312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765575.851549:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765576.874226:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765577.912898:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765578.921999:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765579.946030:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765580.585948:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.585957:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765580.585960:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.585969:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.585978:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.585985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765580.586020:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068029440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586046:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586050:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765580.586052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586055:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586058:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586061:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586070:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765580.586079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068029568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586088:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765580.586093:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586099:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586101:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586105:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765580.586112:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068029696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586120:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586123:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765580.586125:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586128:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586131:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586134:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586137:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765580.586144:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068029824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586152:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586155:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765580.586156:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586159:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765580.586174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068029952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586197:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765580.586199:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586203:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586205:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586208:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586213:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765580.586221:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068030080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765580.586235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765580.586238:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765580.586240:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765580.586243:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3048 00000100:00080000:2.0:1789765580.586246:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765580.586248:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765580.586251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765580.586259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068030208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765580.586336:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068029440:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765580.586389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586434:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061dca80 x1876702068029440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586450:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068029440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586463:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068029440:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765580.586476:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765580.586498:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061de680 x1876702068029568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068029568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586532:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765580.586541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765580.586556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061dfb80 x1876702068029696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068029696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765580.586594:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765580.586609:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586618:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061df800 x1876702068029824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586628:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068029824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765580.586645:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765580.586659:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586667:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061dd180 x1876702068029952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068029952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068029952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765580.586695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765580.586710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061dce00 x1876702068030080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068030080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586742:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765580.586752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765580.586766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765580.586774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765580/real 1789765580] req@ffff9360061dc000 x1876702068030208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765580.586784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068030208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765580.586791:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765580.586794:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765580.586826:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765580.970399:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765581.999788:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765583.024871:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765584.042087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765585.066063:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765585.706208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765585.706222:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706232:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706243:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706261:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765585.706300:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068030336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706340:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765585.706342:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706346:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706350:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706370:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706379:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765585.706392:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068030464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706402:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765585.706408:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706412:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706415:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706419:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706423:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765585.706433:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068030592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706442:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765585.706447:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706451:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706454:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706457:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706461:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765585.706469:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068030720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765585.706482:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706485:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706487:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706491:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706495:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765585.706504:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068030848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706511:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765585.706516:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706519:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706522:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706525:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706529:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765585.706538:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068030976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765585.706547:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765585.706550:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765585.706551:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765585.706555:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3053 00000100:00080000:3.0:1789765585.706558:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765585.706561:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765585.706565:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765585.706587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068031104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765585.707276:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068030336:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765585.707333:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707427:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa6680 x1876702068030336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068030336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707452:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707456:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068030336:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765585.707466:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765585.707487:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707498:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa7480 x1876702068030464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068030464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707518:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707521:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765585.707526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765585.707542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707552:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa5f80 x1876702068030592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707563:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068030592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707571:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707573:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707576:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765585.707581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765585.707599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa4a80 x1876702068030720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707619:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068030720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765585.707639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765585.707658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa5880 x1876702068030848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068030848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068030848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765585.707696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765585.707713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.707723:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa5c00 x1876702068030976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.707733:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068030976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.707740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.707743:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.707746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068030976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765585.707751:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068031104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765585.707767:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765585.709067:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765585/real 1789765585] req@ffff936010aa7800 x1876702068031104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765585.709085:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068031104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765585.709094:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765585.709098:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765585.709101:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068031104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765586.093288:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765587.114071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765588.140960:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765589.162434:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765590.185994:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765590.826304:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826315:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765590.826319:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826329:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826338:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826356:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765590.826395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068031232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826424:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826427:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765590.826429:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826433:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826436:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826440:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826449:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765590.826458:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068031360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826467:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826470:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765590.826471:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826474:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826477:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826481:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826485:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765590.826493:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068031488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826504:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765590.826505:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826508:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826511:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826514:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826518:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765590.826526:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068031616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826536:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765590.826538:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826541:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826543:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826546:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826549:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765590.826556:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068031744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765590.826567:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826569:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826572:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826575:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826579:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765590.826587:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068031872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765590.826596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765590.826599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765590.826600:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765590.826604:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3058 00000100:00080000:2.0:1789765590.826606:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765590.826609:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765590.826613:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765590.826620:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068032000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765590.826770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068031232:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765590.826855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.826894:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061dd500 x1876702068031232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.826911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068031232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.826921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.826925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.826929:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068031232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765590.826936:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765590.826955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.826967:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061df480 x1876702068031360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.826977:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068031360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.826984:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.826987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.826990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765590.826995:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765590.827009:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.827018:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061df100 x1876702068031488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.827028:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068031488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.827035:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.827037:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.827040:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765590.827045:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765590.827060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.827069:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061dd880 x1876702068031616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.827079:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068031616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.827086:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.827089:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.827092:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765590.827096:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765590.827123:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.827133:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061ddc00 x1876702068031744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.827143:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068031744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.827150:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.827152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.827155:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068031744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765590.827160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068031872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765590.827178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.827187:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061de300 x1876702068031872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.827204:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068031872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.827210:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.827213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.827226:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068031872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765590.827231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765590.827249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765590.827260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765590/real 1789765590] req@ffff9360061dc380 x1876702068032000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765590.827270:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068032000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765590.827277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765590.827280:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765590.827283:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765591.209975:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765592.234091:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765593.259573:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765594.282065:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765595.306034:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765595.946663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946672:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765595.946676:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946685:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946703:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946710:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765595.946746:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068032128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.946781:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946785:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765595.946787:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946791:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946794:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946846:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.946854:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765595.946865:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068032256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.946874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765595.946878:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946881:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946885:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946888:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.946892:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765595.946901:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068032384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.946909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946912:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765595.946913:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946917:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946920:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946924:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.946927:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765595.946936:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068032512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.946943:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946946:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765595.946947:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946950:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946953:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946955:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.946958:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765595.946967:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068032640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.946973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.946976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765595.946989:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.946992:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.946995:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.946998:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.947001:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765595.947009:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068032768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765595.947021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765595.947023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765595.947024:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765595.947027:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3063 00000100:00080000:1.0:1789765595.947029:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765595.947031:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765595.947034:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765595.947041:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068032896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765595.947181:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068032128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765595.947252:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765595.947268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765595.947279:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765595.947289:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765595.947299:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765595.947309:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765596.330431:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765597.357347:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765598.377976:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765599.018310:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765599.018349:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765599.018370:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018410:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018420:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018426:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018433:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018439:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765599.018446:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765599.020471:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef4a80 x1876702068032128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068032128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020527:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068032128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765599.020534:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef5880 x1876702068032256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020546:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068032256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020554:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020557:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020560:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765599.020565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef4380 x1876702068032384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068032384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020591:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020594:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020597:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765599.020603:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef4000 x1876702068032512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068032512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020627:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765599.020638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef4700 x1876702068032640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020651:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068032640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068032640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765599.020670:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef5c00 x1876702068032768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068032768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020690:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020693:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020696:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765599.020701:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765595/real 1789765599] req@ffff936010ef5180 x1876702068032896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765599.020712:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068032896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765599.020718:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765599.020721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765599.020724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068032896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765599.402918:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765600.427696:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765601.066081:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765601.066091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765601.066094:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066104:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066114:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066121:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765601.066153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068033024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765601.066188:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765601.066190:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066195:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066198:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066202:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765601.066212:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765601.066225:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068033152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066235:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765601.066239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765601.066240:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066244:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066247:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066250:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765601.066255:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765601.066266:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068033280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765601.066294:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068033024:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765601.066295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765601.066298:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066306:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066310:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066313:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765601.066319:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765601.066331:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068033408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765601.066344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765601.066347:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066351:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066354:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066358:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765601.066363:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765601.066372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765601.066375:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068033536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066386:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765601.066389:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765601.066391:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066395:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066399:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066403:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765601.066408:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765601.066412:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061dc700 x1876702068033024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765601.066421:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068033664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765601.066433:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765601.066436:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068033024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765601.066438:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765601.066442:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765601.066450:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3068 00000100:00080000:2.0:1789765601.066456:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765601.066459:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789765601.066461:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789765601.066464:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765601.066468:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.066474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068033024:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765601.066479:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068033792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765601.066486:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765601.066518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.066531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061ddf80 x1876702068033152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.066543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068033152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.066551:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.066554:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.066557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765601.066563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765601.066581:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.066592:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061dc380 x1876702068033280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.066603:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068033280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.066611:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.066614:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.066617:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765601.066643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765601.066662:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.066673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061de300 x1876702068033408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.066683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068033408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.066691:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.066694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.066697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765601.066702:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765601.066719:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.066728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061ddc00 x1876702068033536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.066738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068033536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.066746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.066749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.066752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068033536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765601.066757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068033664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765601.066774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.068612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061dd880 x1876702068033664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.068646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068033664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.068658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.068662:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.068666:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068033664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765601.068675:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068033792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765601.068722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765601.068739:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765601/real 1789765601] req@ffff9360061df100 x1876702068033792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765601.068749:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068033792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765601.068757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765601.068760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765601.068763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068033792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765601.450028:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765602.475828:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765603.499360:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765604.522897:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765605.546000:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765606.186060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186069:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765606.186073:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186083:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186094:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186101:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765606.186139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068033920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765606.186168:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186172:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186175:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186179:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186188:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765606.186199:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068034048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765606.186213:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186216:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186220:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186223:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186227:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765606.186236:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068034176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186247:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765606.186249:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186252:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186254:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186258:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186261:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765606.186270:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068034304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186278:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186281:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765606.186283:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186286:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186289:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186292:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186307:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765606.186319:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068034432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186327:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186330:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765606.186332:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186335:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186338:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186342:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186345:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765606.186352:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068034560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765606.186361:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765606.186364:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765606.186365:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765606.186369:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3074 00000100:00080000:1.0:1789765606.186440:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765606.186446:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765606.186451:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765606.186521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068034688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765606.186783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068033920:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765606.186930:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.187039:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef7100 x1876702068033920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.187058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068033920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.187068:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.187073:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.187077:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068033920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765606.187084:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765606.187107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.187120:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef4e00 x1876702068034048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.187132:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068034048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.187140:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.187144:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.187147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765606.187153:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765606.187172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.187182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef6d80 x1876702068034176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.187194:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068034176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.187202:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.187205:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.187208:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765606.187214:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765606.187234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.187244:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef6300 x1876702068034304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.187255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068034304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.187263:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.187266:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.187269:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765606.187275:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765606.187303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.187314:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef7b80 x1876702068034432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.187326:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068034432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.187333:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.187336:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.187339:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765606.187344:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068034560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765606.187363:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.188523:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef7480 x1876702068034560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.188554:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068034560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.188564:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.188568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.188672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068034560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765606.188681:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068034688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765606.188786:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765606.188816:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765606/real 1789765606] req@ffff936010ef6680 x1876702068034688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765606.189009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068034688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765606.189019:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765606.189023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765606.189027:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068034688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765606.570725:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765607.594032:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765608.618189:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765609.642018:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765610.666911:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765611.306099:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765611.306113:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306123:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306132:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306139:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765611.306173:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068034816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306204:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765611.306210:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306214:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306218:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306221:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306230:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765611.306243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068034944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765611.306257:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306261:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306264:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306268:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765611.306282:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068035072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306294:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765611.306295:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306299:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306302:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306305:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306309:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765611.306319:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068035200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306331:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765611.306333:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306336:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306339:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306343:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306347:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765611.306357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068035328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306370:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765611.306372:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306375:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306378:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306382:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306386:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765611.306395:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068035456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765611.306404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765611.306407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765611.306408:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765611.306412:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3079 00000100:00080000:1.0:1789765611.306415:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765611.306419:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765611.306422:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765611.306431:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068035584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765611.309120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068034816:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765611.309192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309234:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef7800 x1876702068034816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309253:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068034816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309272:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068034816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765611.309281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765611.309304:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309316:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef5500 x1876702068034944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309327:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068034944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309335:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068034944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765611.309348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765611.309376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef5f80 x1876702068035072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068035072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765611.309418:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765611.309437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef6a00 x1876702068035200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068035200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309466:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309470:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309473:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765611.309479:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765611.309496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309506:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef6680 x1876702068035328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068035328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309524:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309527:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309530:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765611.309536:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068035456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765611.309554:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef7480 x1876702068035456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068035456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068035456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765611.309594:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068035584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765611.309613:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765611.309624:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765611/real 1789765611] req@ffff936010ef7b80 x1876702068035584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765611.309636:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068035584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765611.309644:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765611.309647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765611.309650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068035584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765611.690080:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765612.713999:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765613.738019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765614.762035:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765615.786847:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765616.426769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.426780:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765616.426784:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.426794:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.426825:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.426833:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765616.426874:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068035712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.426903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.426906:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765616.426908:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.426911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.426915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.426918:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.426934:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765616.426948:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068035840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.426956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.426959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765616.426960:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.426964:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.426966:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.426969:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.426973:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765616.426986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068035968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.426994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.426997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765616.426999:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.427002:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.427005:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.427008:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.427012:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765616.427019:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068036096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.427027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.427030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765616.427032:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.427035:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.427038:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.427040:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.427045:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765616.427053:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068036224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.427061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.427064:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765616.427066:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.427069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.427072:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.427075:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.427079:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765616.427087:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068036352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765616.427096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765616.427099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765616.427100:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765616.427104:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3084 00000100:00080000:0.0:1789765616.427107:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765616.427110:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765616.427114:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765616.427124:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068036480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765616.427212:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068035712:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765616.427272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427311:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020daf480 x1876702068035712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427327:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068035712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427340:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427343:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068035712:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765616.427350:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765616.427371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020dafb80 x1876702068035840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427392:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068035840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427400:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427406:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765616.427411:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765616.427427:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020dac380 x1876702068035968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427447:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068035968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427454:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068035968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765616.427464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765616.427481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020dad180 x1876702068036096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068036096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765616.427520:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765616.427539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427617:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020dae300 x1876702068036224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068036224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427640:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427643:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765616.427649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068036352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765616.427668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427677:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020dace00 x1876702068036352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068036352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427694:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427697:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427700:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068036352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765616.427705:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068036480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765616.427721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765616.427730:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765616/real 1789765616] req@ffff936020daea00 x1876702068036480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765616.427741:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068036480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765616.427748:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765616.427751:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765616.427754:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068036480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765616.809999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765617.835839:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765618.859412:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765619.884049:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765620.909067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765621.545923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.545934:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765621.545938:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.545949:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.545959:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.545966:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765621.546002:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068036608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546036:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546040:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765621.546042:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546046:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546050:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546053:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765621.546074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068036736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765621.546088:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546092:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546095:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546098:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546103:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765621.546113:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068036864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546131:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546134:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765621.546135:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546139:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546141:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546145:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546149:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765621.546159:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068036992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546171:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546174:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765621.546175:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546178:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546180:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546183:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765621.546196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068037120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546205:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765621.546207:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546209:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546212:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765621.546226:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068037248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765621.546233:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765621.546237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765621.546238:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765621.546241:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3089 00000100:00080000:0.0:1789765621.546244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765621.546247:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765621.546252:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765621.546261:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068037376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765621.546763:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068036608:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765621.546846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.546900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020dadf80 x1876702068036608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.546917:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068036608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.546926:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.546929:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.546933:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068036608:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765621.546939:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765621.546962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.546974:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020dadc00 x1876702068036736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.546985:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068036736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.546992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.546994:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.546997:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765621.547005:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765621.547024:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.547037:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020daf800 x1876702068036864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.547047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068036864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.547054:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.547056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.547059:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765621.547064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765621.547082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.547092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020daea00 x1876702068036992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.547102:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068036992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.547108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.547111:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.547113:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068036992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765621.547131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765621.547150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.547159:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020dace00 x1876702068037120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.547169:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068037120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.547175:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.547178:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.547181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765621.547185:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068037248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765621.547202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.547212:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020dae300 x1876702068037248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.547222:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068037248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.547228:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.547231:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.547234:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068037248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765621.547240:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068037376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765621.547258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765621.547267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765621/real 1789765621] req@ffff936020dad180 x1876702068037376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765621.547276:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068037376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765621.547288:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765621.547291:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765621.547294:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068037376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765621.931358:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765622.955178:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765623.979484:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765625.004303:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765626.026018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765626.667448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.667458:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765626.667461:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.667471:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.667480:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.667486:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765626.667519:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068037504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765626.667597:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068037504:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765626.667655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765626.667686:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.667689:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765626.667691:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.667695:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.667700:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.667703:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765626.667712:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765626.667724:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068037632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765626.671709:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061df480 x1876702068037504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.671737:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068037504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.671748:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.671752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.671756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068037504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765626.671795:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765626.671874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765626.671891:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061dd500 x1876702068037632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.671903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068037632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.671916:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.671920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.672410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037632:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765626.672705:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.672710:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765626.672713:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.672718:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.672723:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.672730:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765626.672740:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765626.672766:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068037760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765626.672874:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765626.672905:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765626.672947:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.672951:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765626.672953:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.672957:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.672961:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.672966:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765626.672971:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765626.672983:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068037888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765626.677434:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061dc000 x1876702068037760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.677463:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068037760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.677472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.677476:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.677481:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765626.677509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765626.677565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765626.677580:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061dce00 x1876702068037888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.677592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068037888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.677600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.677604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.677607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068037888:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789765626.677700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.677705:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765626.677707:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.677711:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.677716:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.677722:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765626.677732:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765626.677754:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068038016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765626.677776:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.677780:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765626.677782:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.677786:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.678002:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.678006:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765626.678012:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765626.678025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068038144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765626.678034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765626.678037:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765626.678038:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765626.678041:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3094 00000100:00080000:2.0:1789765626.678044:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765626.678048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765626.678440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765626.678472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765626.678487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061dd180 x1876702068038016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.678499:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068038016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.678506:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.678509:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.678511:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765626.678517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068038144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765626.678537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765626.678548:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061df800 x1876702068038144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.678559:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068038144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.678566:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.678569:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.678572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068038144:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789765626.678618:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765626.678633:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068038272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765626.678910:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068038272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765626.679128:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765626.679141:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765626/real 1789765626] req@ffff9360061dfb80 x1876702068038272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765626.679152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068038272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765626.679159:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765626.679162:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765626.679165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068038272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765627.051149:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765628.074186:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765629.098042:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765630.124437:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765631.146079:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765631.786703:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.786713:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765631.786717:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.786729:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.786739:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.786745:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765631.786781:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068038400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765631.786931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068038400:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765631.786990:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765631.787034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.787038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765631.787040:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.787044:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.787048:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.787052:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.787071:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765631.787084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068038528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765631.790052:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061de680 x1876702068038400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.790089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068038400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.790100:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.790104:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.790117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068038400:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765631.790149:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765631.790198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.790215:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061dca80 x1876702068038528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.790229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068038528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.790237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.790240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.790244:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038528:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765631.790442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.790447:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765631.790460:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.790465:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.790471:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.790476:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.790486:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765631.790509:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068038656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765631.790522:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.790525:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765631.790527:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.790530:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.790534:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.790538:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.790542:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765631.790552:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068038784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765631.790561:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.790564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765631.790565:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.790569:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.790572:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.790575:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.790579:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765631.790589:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068038912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765631.790597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.790601:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765631.790602:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.790606:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.790609:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.790612:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.790616:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765631.790626:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068039040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765631.790641:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765631.790656:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765631.790658:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765631.790661:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3099 00000100:00080000:2.0:1789765631.790665:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765631.790668:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765631.790673:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765631.790682:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068039168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765631.791373:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765631.791405:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.791421:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061dca80 x1876702068038656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.791433:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068038656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.791441:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.791444:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.791510:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765631.791519:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765631.791543:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.791554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061de680 x1876702068038784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.791565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068038784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.791576:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.791579:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.791582:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765631.791587:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765631.791606:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.791617:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061dfb80 x1876702068038912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.791627:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068038912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.791634:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.791637:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.791640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068038912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765631.791646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765631.791665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.791676:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061df800 x1876702068039040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.791687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068039040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.791880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.791884:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.791887:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765631.791893:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765631.791916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765631.791927:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765631/real 1789765631] req@ffff9360061dd180 x1876702068039168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765631.791938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068039168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765631.791945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765631.791948:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765631.791951:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765632.171196:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765633.194132:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765634.220040:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765635.242223:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765636.266605:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765636.908591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765636.908603:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908612:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908621:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908627:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765636.908660:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068039296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765636.908695:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908699:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908702:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765636.908713:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765636.908724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068039424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765636.908736:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908747:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765636.908750:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765636.908760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068039552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908781:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765636.908783:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908787:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908791:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908794:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765636.908815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765636.908829:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068039680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765636.908844:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765636.908845:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068039296:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765636.908857:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908863:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908867:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765636.908872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765636.908884:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068039808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908892:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765636.908895:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765636.908897:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908899:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908903:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789765636.908906:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765636.908910:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765636.908915:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765636.908936:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068039936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765636.908945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765636.908949:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa4000 x1876702068039296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765636.908950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765636.908954:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765636.908963:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3104 00000100:00080000:3.0:1789765636.908968:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765636.908972:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765636.908976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068039296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789765636.908979:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765636.908992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789765636.908995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068040064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765636.908999:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.909006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068039296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765636.909018:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765636.909041:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.909054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa7b80 x1876702068039424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.909072:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068039424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.909156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.909160:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.909164:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765636.909170:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765636.909193:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.909205:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa6300 x1876702068039552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.909938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068039552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.909949:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.909953:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.909956:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765636.910007:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765636.910033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.910046:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa7800 x1876702068039680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.910057:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068039680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.910072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.910085:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.910089:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765636.910095:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765636.910112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.910121:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa5c00 x1876702068039808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.910132:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068039808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.910139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.910143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.910146:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068039808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765636.910152:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765636.910167:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.910176:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa5880 x1876702068039936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.910186:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068039936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.910194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.910197:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.910200:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068039936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765636.910206:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765636.910221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765636.910230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765636/real 1789765636] req@ffff936010aa4a80 x1876702068040064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765636.910240:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068040064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765636.910248:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765636.910251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765636.910254:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765637.290139:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765638.314643:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765639.339397:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765640.363705:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765641.386614:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765642.027841:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.027851:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765642.027855:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.027866:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.027877:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.027884:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765642.027919:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068040192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765642.028073:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028078:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028082:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028095:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765642.028109:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068040320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765642.028123:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028127:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028130:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028133:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028136:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765642.028146:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068040448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765642.028158:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028161:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028168:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028171:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765642.028179:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068040576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765642.028190:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028193:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028196:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028198:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028202:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765642.028210:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068040704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765642.028223:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028226:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028229:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028232:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028236:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765642.028247:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068040832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765642.028261:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765642.028265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765642.028266:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765642.028270:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3109 00000100:00080000:1.0:1789765642.028273:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765642.028277:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765642.028281:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765642.028289:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068040960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765642.028379:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068040192:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765642.028444:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef6300 x1876702068040192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068040192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028522:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068040192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765642.028530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765642.028553:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028566:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef6d80 x1876702068040320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028578:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068040320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028585:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028588:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028591:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765642.028596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765642.028615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef4e00 x1876702068040448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068040448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765642.028657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765642.028676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef7100 x1876702068040576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068040576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028716:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765642.028725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765642.028756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef5180 x1876702068040704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028779:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068040704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028787:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028790:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028793:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068040704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765642.028826:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765642.028847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef5c00 x1876702068040832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068040832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.028883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.028887:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.028890:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765642.028896:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765642.028914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765642.028925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765642/real 1789765642] req@ffff936010ef4700 x1876702068040960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765642.028936:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068040960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765642.029014:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765642.029018:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765642.029022:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068040960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765642.412596:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765643.436519:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765644.457999:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765645.482059:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765646.506269:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765647.146275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765647.146290:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146301:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146311:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146319:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765647.146357:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068041088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146390:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146394:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765647.146395:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146400:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146403:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146407:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146416:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765647.146428:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068041216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146440:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765647.146442:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146445:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146448:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146452:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146455:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765647.146464:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068041344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146475:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765647.146477:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146481:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146484:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146488:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146492:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765647.146501:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068041472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146510:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765647.146515:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146518:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146520:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146524:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146527:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765647.146536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068041600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765647.146547:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146550:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146553:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146556:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146560:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765647.146569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068041728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765647.146577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765647.146580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765647.146582:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765647.146585:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3114 00000100:00080000:3.0:1789765647.146588:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765647.146591:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765647.146596:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765647.146604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068041856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765647.146780:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068041088:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765647.146864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.146916:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa5f80 x1876702068041088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147560:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068041088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.147574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.147578:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.147583:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068041088:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765647.147591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765647.147622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.147636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa7480 x1876702068041216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147649:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068041216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.147656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.147660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.147663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765647.147669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765647.147689:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.147714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa6680 x1876702068041344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147726:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068041344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.147734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.147737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.147740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765647.147761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765647.147776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.147858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa5180 x1876702068041472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147870:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068041472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.147878:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.147882:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.147885:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765647.147891:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765647.147912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.147923:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa4700 x1876702068041600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147934:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068041600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.147942:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.147945:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.147948:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068041600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765647.147953:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068041728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765647.147973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.147984:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa4380 x1876702068041728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.147995:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068041728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.148002:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.148005:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.148008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068041728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765647.148014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068041856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765647.148033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765647.148044:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765647/real 1789765647] req@ffff936010aa4e00 x1876702068041856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765647.148054:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068041856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765647.148061:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765647.148064:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765647.148067:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068041856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765647.530526:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765648.555758:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765649.577964:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765650.602185:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765651.628354:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765652.266089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765652.266104:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266115:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266124:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266132:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765652.266167:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068041984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266197:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266201:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765652.266246:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266251:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266255:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266259:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266267:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765652.266278:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068042112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266297:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266300:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765652.266301:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266305:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266308:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266312:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765652.266324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068042240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765652.266337:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266340:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266343:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266346:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266350:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765652.266357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068042368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266364:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266367:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765652.266369:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266372:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266374:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266377:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266381:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765652.266389:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068042496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266396:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266399:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765652.266401:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266404:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266406:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266409:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266413:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765652.266420:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068042624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765652.266428:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765652.266442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765652.266443:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765652.266447:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3120 00000100:00080000:2.0:1789765652.266449:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765652.266453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765652.266457:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765652.266465:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068042752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765652.266916:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068041984:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765652.266972:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267020:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061dce00 x1876702068041984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267038:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068041984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267048:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267052:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267055:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068041984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765652.267062:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765652.267084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267095:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061dc000 x1876702068042112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068042112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765652.267121:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765652.267140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061dd500 x1876702068042240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068042240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267168:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267171:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765652.267179:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765652.267199:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267222:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061df480 x1876702068042368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267233:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068042368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267241:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765652.267253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765652.267271:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267281:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061df100 x1876702068042496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267292:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068042496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267299:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267302:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267305:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068042496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765652.267310:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068042624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765652.267329:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267339:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061dd880 x1876702068042624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267366:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068042624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267377:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267380:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068042624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765652.267384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068042752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765652.267403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765652.267414:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765652/real 1789765652] req@ffff9360061ddc00 x1876702068042752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765652.267425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068042752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765652.267432:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765652.267435:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765652.267438:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068042752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765652.650179:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765653.674258:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765654.698839:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765655.724760:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765656.747476:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765657.391376:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391385:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765657.391390:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391400:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391410:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391417:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765657.391450:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068042880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391481:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765657.391486:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391491:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391494:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391497:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391506:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765657.391518:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068043008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391528:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391531:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765657.391533:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391537:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391540:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391543:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391547:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765657.391556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068043136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391564:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765657.391569:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391572:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391575:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391579:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391583:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765657.391593:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068043264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391601:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391604:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765657.391605:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391609:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391612:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391616:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391619:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765657.391639:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068043392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765657.391653:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391656:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391660:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391663:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391667:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765657.391677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068043520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765657.391685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765657.391688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765657.391690:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765657.391693:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3125 00000100:00080000:3.0:1789765657.391696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765657.391700:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765657.391704:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765657.391712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068043648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765657.391826:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068042880:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765657.391881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.391932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa6a00 x1876702068042880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.391947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068042880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.391957:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.391961:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.391964:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068042880:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765657.391970:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765657.391992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392005:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa5500 x1876702068043008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392028:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068043008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392037:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392040:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765657.392049:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765657.392068:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa7100 x1876702068043136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068043136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392098:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392101:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765657.392110:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765657.392130:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa4e00 x1876702068043264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392150:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068043264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392157:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392160:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392162:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765657.392167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765657.392185:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392195:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa4380 x1876702068043392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068043392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392213:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392216:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392219:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765657.392224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068043520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765657.392244:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392253:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa4700 x1876702068043520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392263:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068043520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392270:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392272:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392275:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068043520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765657.392280:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068043648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765657.392294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765657.392303:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765657/real 1789765657] req@ffff936010aa5180 x1876702068043648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765657.392313:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068043648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765657.392319:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765657.392321:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765657.392324:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068043648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765657.770378:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765658.809084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765659.818527:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765660.846172:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765661.866976:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765662.509047:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765662.509060:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509069:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509078:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509085:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765662.509119:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068043776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765662.509151:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509154:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509163:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509176:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765662.509186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068043904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765662.509199:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509203:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509206:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509209:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509212:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765662.509220:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068044032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765662.509232:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509235:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509248:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509251:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509255:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765662.509264:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068044160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765662.509276:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509279:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509281:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509284:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509288:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765662.509295:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068044288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765662.509307:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509310:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509312:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509315:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509318:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765662.509325:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068044416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765662.509546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765662.509549:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765662.509550:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765662.509554:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3130 00000100:00080000:1.0:1789765662.509558:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765662.509561:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765662.509566:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765662.509576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068044544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765662.510518:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068043776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765662.510625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765662.510652:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765662.510674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765662.510696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765662.510716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068044416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765662.510734:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068044544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765662.890042:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765663.914181:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765664.938006:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765665.578317:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765665.578373:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765665.578391:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578429:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578438:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578445:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578452:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578459:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765665.578466:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765665.579077:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef4000 x1876702068043776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579105:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068043776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579117:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579126:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068043776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765665.579134:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef4380 x1876702068043904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068043904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579153:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579157:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579160:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068043904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765665.579166:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef5880 x1876702068044032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579182:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068044032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579189:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579192:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579194:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765665.579201:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef4a80 x1876702068044160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068044160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579217:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579220:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765665.579229:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef4700 x1876702068044288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579240:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068044288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579250:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579252:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765665.579258:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef5c00 x1876702068044416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579267:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068044416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579273:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579276:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579279:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068044416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765665.579284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765662/real 1789765665] req@ffff936010ef5180 x1876702068044544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765665.579294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068044544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765665.579300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765665.579303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765665.579306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068044544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765665.968645:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765666.985970:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765667.625911:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.625919:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765667.625922:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.625930:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.625937:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.625943:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765667.625969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068044672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.625991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.625994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765667.625995:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.625998:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626000:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.626002:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626009:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765667.626020:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068044800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.626028:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.626032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765667.626033:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.626036:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626039:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.626042:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626047:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765667.626056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068044928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.626065:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.626068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765667.626070:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.626073:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626076:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.626079:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626084:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765667.626094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068045056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.626104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.626107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765667.626109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.626112:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626117:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765667.626118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068044672:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765667.626123:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626130:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765667.626142:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068045184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.626151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.626154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765667.626156:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.626160:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626164:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.626167:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626171:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765667.626182:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765667.626182:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068045312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765667.626193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765667.626197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765667.626199:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765667.626203:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3135 00000100:00080000:3.0:1789765667.626206:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765667.626209:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765667.626213:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765667.626221:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa6680 x1876702068044672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765667.626225:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068045440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765667.626256:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068044672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.626268:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.626272:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.626276:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068044672:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765667.626283:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765667.626305:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.626318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa7480 x1876702068044800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.626329:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068044800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.626336:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.626339:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.626342:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765667.626347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765667.626379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.627596:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa5f80 x1876702068044928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.627622:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068044928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.627632:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.627636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.627639:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068044928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765667.627671:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765667.627903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.627919:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa4a80 x1876702068045056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.627931:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068045056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.627939:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.627942:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.627946:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765667.627952:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765667.627971:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.627982:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa5880 x1876702068045184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.627993:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068045184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.627999:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.628002:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.628005:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765667.628011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068045312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765667.628030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.628041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa5c00 x1876702068045312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.628051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068045312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.628059:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.628062:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.628065:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068045312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765667.628071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068045440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765667.628088:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765667.628098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765667/real 1789765667] req@ffff936010aa7800 x1876702068045440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765667.628590:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068045440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765667.628598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765667.628602:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765667.628605:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068045440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765668.010344:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765669.037829:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765670.059854:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765671.082098:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765672.106334:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765672.748830:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.748840:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765672.748844:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.748852:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.748862:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.748869:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765672.748900:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068045568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.748923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.748927:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765672.748928:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.748932:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.748935:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.748937:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.748946:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765672.748956:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068045696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.748965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.748969:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765672.748972:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765672.748973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068045568:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765672.748978:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.748984:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.748987:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.748991:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765672.749003:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068045824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.749012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.749016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765672.749018:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.749021:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.749024:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.749028:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.749032:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789765672.749038:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789765672.749043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068045952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.749053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.749057:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765672.749058:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.749062:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.749066:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.749070:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.749074:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765672.749084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068046080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.749091:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.749094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765672.749095:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.749098:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00080000:2.0:1789765672.749101:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.749105:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.749109:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765672.749118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068046208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765672.749126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765672.749129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765672.749130:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765672.749134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3140 00000100:00100000:0.0:1789765672.749139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061de300 x1876702068045568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765672.749140:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765672.749148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765672.749156:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765672.749165:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068045568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765672.749169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068046336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765672.749187:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749193:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068045568:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765672.749219:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765672.749240:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749250:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061dc380 x1876702068045696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068045696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749269:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749271:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749274:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765672.749279:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765672.749294:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749302:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061ddf80 x1876702068045824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749312:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068045824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749320:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749358:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749382:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765672.749389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765672.749477:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749504:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061dc700 x1876702068045952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068045952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749538:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749540:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068045952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765672.749548:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765672.749565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061ddc00 x1876702068046080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749592:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068046080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749606:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765672.749619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068046208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765672.749635:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061dd880 x1876702068046208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068046208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068046208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765672.749687:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068046336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765672.749702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765672.749711:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765672/real 1789765672] req@ffff9360061df100 x1876702068046336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765672.749722:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068046336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765672.749738:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765672.749741:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765672.749744:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068046336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765673.130067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765674.156190:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765675.178147:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765676.202164:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765677.231124:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765677.869638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765677.869653:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869663:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869674:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869681:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765677.869716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068046464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869757:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765677.869759:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869767:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869770:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.869780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765677.869790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068046592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869825:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765677.869830:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869834:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869838:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.869846:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765677.869857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068046720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869869:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765677.869871:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869874:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869877:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869880:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.869884:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765677.869892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068046848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869900:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765677.869905:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869909:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869911:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869915:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.869918:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765677.869926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068046976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765677.869952:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869955:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869958:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.869962:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.869966:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765677.869976:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068047104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765677.869985:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765677.869988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765677.869989:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765677.869993:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3145 00000100:00080000:0.0:1789765677.869997:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765677.870000:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765677.870004:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765677.870013:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068047232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765677.871146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068046464:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765677.871217:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020dac380 x1876702068046464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871305:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068046464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871319:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068046464:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765677.871331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765677.871354:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871366:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020dafb80 x1876702068046592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068046592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871386:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871392:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765677.871401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765677.871422:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020daf480 x1876702068046720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871445:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068046720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871453:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871456:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871459:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765677.871465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765677.871497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020daf100 x1876702068046848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068046848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871529:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871532:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765677.871542:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765677.871561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871572:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020dac700 x1876702068046976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871584:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068046976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871591:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871595:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068046976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765677.871604:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068047104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765677.871625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020daed80 x1876702068047104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068047104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871661:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068047104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765677.871667:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068047232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765677.871688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765677.871699:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765677/real 1789765677] req@ffff936020dae680 x1876702068047232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765677.871710:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068047232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765677.871724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765677.871727:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765677.871730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068047232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765678.250061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765679.274211:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765680.298003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765681.325146:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765682.346155:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765682.986127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765682.986139:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986149:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986159:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986166:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765682.986202:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068047360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765682.986238:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986242:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986245:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986248:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986257:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765682.986269:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068047488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986276:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765682.986281:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986284:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986287:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986292:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986296:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765682.986304:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068047616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765682.986317:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986320:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986323:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986326:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986330:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765682.986338:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068047744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986346:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986348:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765682.986350:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986353:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986356:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986359:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986363:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765682.986373:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068047872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986381:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986384:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765682.986386:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986389:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986392:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986396:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986400:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765682.986409:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068048000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765682.986417:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765682.986420:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765682.986421:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765682.986425:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3150 00000100:00080000:1.0:1789765682.986427:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765682.986430:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765682.986445:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765682.986455:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068048128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765682.986544:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068047360:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765682.986598:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986645:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef7100 x1876702068047360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986662:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068047360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.986672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.986675:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.986678:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068047360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765682.986684:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765682.986707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef4e00 x1876702068047488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068047488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.986742:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.986744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.986747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765682.986752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765682.986770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef6d80 x1876702068047616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986791:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068047616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.986824:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.986827:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.986831:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765682.986836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765682.986858:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef6300 x1876702068047744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986881:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068047744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.986888:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.986891:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.986894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765682.986900:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765682.986917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986928:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef7b80 x1876702068047872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986939:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068047872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.986947:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.986950:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.986954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068047872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765682.986960:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765682.986978:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.986988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef7480 x1876702068048000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.986998:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068048000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.987006:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.987009:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.987012:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765682.987018:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765682.987599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765682.987614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765682/real 1789765682] req@ffff936010ef6680 x1876702068048128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765682.987627:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068048128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765682.987635:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765682.987638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765682.987640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765683.370171:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765684.398844:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765685.418070:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765686.442854:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765687.466535:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765688.106006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.106014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765688.106018:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.106027:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.106037:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.106045:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765688.106113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068048256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765688.106138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.106141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765688.106143:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.106147:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.106150:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.106154:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765688.106163:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765688.106173:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068048384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765688.106182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.106185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765688.106187:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.106190:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.106192:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.106195:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765688.106198:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765688.106206:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068048512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765688.106214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.106313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765688.106315:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765688.106348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068048256:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765688.106415:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.106421:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789765688.106426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765688.106428:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765688.106435:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765688.106448:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068048640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765688.106482:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef6a00 x1876702068048256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.106501:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068048256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.106513:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.106517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.106522:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068048256:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765688.106529:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765688.106553:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.106564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef5f80 x1876702068048384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.106582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068048384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.106600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.106603:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.106607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765688.106613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765688.106634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.106645:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef5500 x1876702068048512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.106657:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068048512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.106663:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.106666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.106669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048512:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765688.106704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.106708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765688.106710:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.106715:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00100000:3.0:1789765688.106719:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048640:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789765688.106721:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.106729:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:3.0:1789765688.106748:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.106760:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef7800 x1876702068048640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.106786:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068048640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789765688.106795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789765688.106821:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.106824:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.106827:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765688.107046:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068048768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765688.107064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.107068:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765688.107070:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.107073:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.107077:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.107082:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765688.107087:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765688.107097:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068048896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765688.107109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765688.107112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765688.107113:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765688.107117:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3155 00000100:00080000:1.0:1789765688.107120:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765688.107123:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765688.107126:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765688.107133:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068049024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765688.107473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765688.107514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.107529:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef7800 x1876702068048768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.107542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068048768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.107550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.107553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.107557:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068048768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765688.107563:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765688.107583:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.107601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef5500 x1876702068048896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.107613:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068048896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.107620:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.107623:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.107627:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068048896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765688.107632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765688.107650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765688.107661:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765688/real 1789765688] req@ffff936010ef5f80 x1876702068049024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765688.107672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068049024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765688.107679:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765688.107682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765688.107685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765688.490126:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765689.516330:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765690.538024:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765691.562181:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765692.585997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765693.227079:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765693.227095:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227105:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227116:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227124:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765693.227161:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068049152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765693.227190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765693.227196:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227200:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227203:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227206:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765693.227215:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765693.227226:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068049280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765693.227236:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227241:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789765693.227241:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068049152:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765693.227244:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227253:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227256:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227259:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765693.227264:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765693.227275:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068049408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765693.227284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227287:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765693.227288:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227292:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227296:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227299:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765693.227303:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789765693.227306:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765693.227317:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068049536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765693.227326:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765693.227330:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227334:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227338:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227342:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765693.227346:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765693.227357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068049664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765693.227375:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020dad500 x1876702068049152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765693.227391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765693.227397:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765693.227402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068049152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765693.227403:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227410:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227417:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:3.0:1789765693.227418:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789765693.227425:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789765693.227425:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068049152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765693.227440:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068049792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765693.227457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049280:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765693.227462:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765693.227466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765693.227468:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765693.227473:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3161 00000100:00080000:0.0:1789765693.227478:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765693.227482:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:3.0:1789765693.227485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789765693.227489:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765693.227502:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068049920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765693.227513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020dad880 x1876702068049280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227527:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068049280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227542:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765693.227547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765693.227565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765693.227575:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020dac000 x1876702068049408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227585:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068049408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227592:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227595:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765693.227604:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765693.227622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765693.227632:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020dae680 x1876702068049536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227643:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068049536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227651:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227655:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227657:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765693.227663:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765693.227681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765693.227692:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020daed80 x1876702068049664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068049664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227711:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227714:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068049664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765693.227736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765693.227754:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765693.227764:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020dac700 x1876702068049792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227773:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068049792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227787:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765693.227794:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765693.227838:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765693.227849:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765693/real 1789765693] req@ffff936020daf100 x1876702068049920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765693.227858:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068049920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765693.227865:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765693.227879:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765693.227888:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068049920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765693.612749:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765694.636399:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765695.658212:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765696.682959:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765697.708843:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765698.347358:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347369:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765698.347372:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347382:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347392:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347399:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765698.347444:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068050048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765698.347481:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347485:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347489:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347492:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347501:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765698.347514:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068050176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347523:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765698.347539:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347544:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347548:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347552:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347557:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765698.347569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068050304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347578:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765698.347583:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347586:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347589:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347591:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347595:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765698.347603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068050432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765698.347615:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347618:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347621:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347624:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347627:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765698.347634:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068050560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765698.347646:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347651:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347654:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347658:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765698.347665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068050688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765698.347673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765698.347676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765698.347678:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765698.347681:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3166 00000100:00080000:1.0:1789765698.347684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765698.347687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765698.347690:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765698.347699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068050816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765698.347886:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068050048:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765698.347944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348007:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef6a00 x1876702068050048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068050048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348044:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348048:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348051:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068050048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765698.348058:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765698.348082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef6680 x1876702068050176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068050176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348123:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348126:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348130:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765698.348135:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765698.348153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348163:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef7480 x1876702068050304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348174:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068050304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348182:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348184:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348187:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765698.348192:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765698.348209:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef7b80 x1876702068050432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068050432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348240:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765698.348247:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765698.348264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348274:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef6300 x1876702068050560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348285:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068050560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348293:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348296:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348299:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068050560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765698.348304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068050688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765698.348330:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348343:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef6d80 x1876702068050688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348353:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068050688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348360:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348365:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068050688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765698.348370:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068050816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765698.348389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765698.348399:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765698/real 1789765698] req@ffff936010ef4e00 x1876702068050816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765698.348421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068050816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765698.348427:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765698.348430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765698.348433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068050816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765698.730016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765699.754248:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765700.783668:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765701.805173:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765702.826490:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765703.467187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765703.467201:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467211:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467221:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467229:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765703.467270:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068050944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467323:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467327:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765703.467329:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467332:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467336:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467340:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467359:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765703.467373:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068051072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467383:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467386:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765703.467388:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467391:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467394:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467398:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467402:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765703.467412:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068051200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467421:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467424:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765703.467426:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467429:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467432:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467436:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467440:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765703.467450:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068051328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765703.467463:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467467:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467470:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467473:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467478:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765703.467487:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068051456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467509:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765703.467511:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467514:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467518:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467521:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467525:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765703.467535:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068051584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765703.467543:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765703.467546:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765703.467548:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765703.467551:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3171 00000100:00080000:0.0:1789765703.467555:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765703.467558:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765703.467562:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765703.467571:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068051712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765703.467760:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068050944:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765703.467846:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.467889:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020daf480 x1876702068050944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.467912:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068050944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.467922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.467925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.467929:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068050944:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765703.467937:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765703.467961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.467973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020dafb80 x1876702068051072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.467983:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068051072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.467990:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.468187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.468191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765703.468198:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765703.468224:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.468236:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020dac380 x1876702068051200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.468247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068051200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.468261:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.468265:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.468268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765703.468274:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765703.468308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.468320:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020dad180 x1876702068051328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.468332:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068051328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.468340:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.468343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.468346:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765703.468352:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765703.468371:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.468382:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020dae300 x1876702068051456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.468394:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068051456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.468401:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.468403:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.468407:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765703.468414:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068051584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765703.468433:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.468444:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020dace00 x1876702068051584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.468455:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068051584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.468462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.468465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.468468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068051584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765703.468474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068051712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765703.471482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765703.471524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765703/real 1789765703] req@ffff936020daea00 x1876702068051712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765703.471547:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068051712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765703.471559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765703.471563:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765703.471567:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068051712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765703.857624:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765704.874541:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765705.897996:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765706.922091:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765707.946007:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765708.588168:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765708.588181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588191:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588202:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588210:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765708.588277:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068051840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588309:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588324:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765708.588327:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588332:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588336:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588340:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588349:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765708.588361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068051968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588372:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588375:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765708.588377:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588381:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588384:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588388:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588393:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765708.588403:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068052096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588413:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765708.588418:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588421:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588424:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588428:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588432:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765708.588443:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068052224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765708.588455:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588458:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588462:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588465:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588469:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765708.588478:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068052352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588499:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765708.588504:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588508:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588511:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588515:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588519:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765708.588528:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068052480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765708.588536:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765708.588539:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765708.588541:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765708.588544:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3176 00000100:00080000:3.0:1789765708.588547:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765708.588551:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765708.588554:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765708.588563:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068052608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765708.588837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068051840:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765708.588903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.588940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa6300 x1876702068051840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.588957:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068051840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.588966:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.588971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.588974:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068051840:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765708.588982:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765708.589003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589015:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa7b80 x1876702068051968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589026:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068051968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589033:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589036:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589039:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068051968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765708.589044:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765708.589066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589077:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa4000 x1876702068052096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068052096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589096:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589099:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589101:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765708.589107:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765708.589127:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589139:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa7800 x1876702068052224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589149:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068052224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589156:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589158:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589161:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765708.589167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765708.589187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589211:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa5c00 x1876702068052352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068052352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589237:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765708.589258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068052480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765708.589278:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589289:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa5880 x1876702068052480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589300:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068052480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589307:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589309:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589312:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068052480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765708.589857:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068052608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765708.589894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765708.589908:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765708/real 1789765708] req@ffff936010aa4a80 x1876702068052608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765708.589919:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068052608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765708.589932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765708.589935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765708.589938:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068052608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765708.971160:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765709.994315:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765711.019927:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765712.043174:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765713.067389:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765713.705961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.705971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765713.705976:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.705984:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.705994:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706001:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765713.706036:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068052736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706066:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765713.706072:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706075:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706079:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706083:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706092:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765713.706103:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068052864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765713.706118:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706122:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706126:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706130:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706134:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765713.706143:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068052992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765713.706157:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706160:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706164:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706167:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706182:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765713.706194:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068053120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765713.706216:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706220:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706224:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706227:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706232:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765713.706241:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068053248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765713.706254:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706257:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706259:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706262:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706266:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765713.706274:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068053376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765713.706280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765713.706283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765713.706284:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765713.706287:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3181 00000100:00080000:0.0:1789765713.706290:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765713.706293:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765713.706296:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765713.706303:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068053504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765713.706465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068052736:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765713.706524:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020daf800 x1876702068052736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068052736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706592:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706596:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068052736:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765713.706606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765713.706627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706639:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020dadc00 x1876702068052864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068052864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706661:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765713.706670:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765713.706688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020dadf80 x1876702068052992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706777:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068052992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068052992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765713.706817:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765713.706840:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706851:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020daea00 x1876702068053120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706863:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068053120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706871:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706874:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706877:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765713.706883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765713.706902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706912:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020dace00 x1876702068053248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068053248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706931:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706934:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706937:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765713.706943:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068053376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765713.706960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.706970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020dae300 x1876702068053376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.706981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068053376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.706988:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.706991:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.706994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068053376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765713.706999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068053504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765713.707017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765713.707027:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765713/real 1789765713] req@ffff936020dad180 x1876702068053504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765713.707038:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068053504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765713.707045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765713.707048:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765713.707051:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068053504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765714.090638:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765715.114094:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765716.140619:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765717.162015:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765718.187424:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765718.825928:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.825937:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765718.825941:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.825950:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.825961:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.825968:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765718.826003:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068053632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765718.826162:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826167:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826179:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826183:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826192:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765718.826207:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068053760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765718.826221:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826224:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826227:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826230:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826234:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765718.826245:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068053888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765718.826256:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826259:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826261:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826265:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826268:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765718.826276:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068054016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826284:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826286:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765718.826288:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826290:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826293:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826295:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826299:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765718.826308:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068054144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826315:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765718.826319:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826323:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826326:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826330:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826334:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765718.826343:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068054272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765718.826351:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765718.826353:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765718.826355:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765718.826357:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3186 00000100:00080000:0.0:1789765718.826360:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765718.826363:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765718.826367:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765718.826375:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068054400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765718.826787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068053632:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765718.826880:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.826917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020dac380 x1876702068053632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.826938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068053632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.826947:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.826951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.826955:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068053632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765718.826962:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765718.826984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.826996:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020dafb80 x1876702068053760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827007:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068053760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827014:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827020:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765718.827025:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765718.827044:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.827055:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020daf480 x1876702068053888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827067:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068053888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827080:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827083:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827086:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068053888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765718.827091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765718.827122:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.827134:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020daf100 x1876702068054016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827144:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068054016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827155:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827158:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765718.827163:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765718.827192:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.827203:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020dac700 x1876702068054144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068054144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827303:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827306:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827309:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765718.827315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068054272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765718.827435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.827448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020daed80 x1876702068054272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068054272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827468:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068054272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765718.827480:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068054400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765718.827500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765718.827510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765718/real 1789765718] req@ffff936020dae680 x1876702068054400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765718.827521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068054400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765718.827528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765718.827531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765718.827534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068054400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765719.210554:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765720.234578:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765721.258373:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765722.283087:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765723.306883:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765723.947451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947461:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765723.947465:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947663:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947674:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947680:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765723.947717:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068054528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947752:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765723.947758:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947762:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947765:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947769:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947778:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765723.947789:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068054656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947839:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765723.947841:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947845:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947848:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947851:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947856:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765723.947866:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068054784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947875:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947878:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765723.947880:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947883:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947886:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947889:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947893:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765723.947902:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068054912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947910:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947913:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765723.947914:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947917:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947920:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947923:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947927:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765723.947934:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068055040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765723.947947:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947949:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947952:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947955:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947958:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765723.947965:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068055168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765723.947974:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765723.947976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765723.947978:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765723.947981:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3191 00000100:00080000:3.0:1789765723.947984:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765723.947987:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765723.947991:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765723.947999:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068055296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765723.948268:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068054528:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765723.948448:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948489:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa5f80 x1876702068054528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948506:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068054528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948516:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948520:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948524:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068054528:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765723.948530:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765723.948551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa7480 x1876702068054656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068054656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948582:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948585:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948588:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765723.948593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765723.948611:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948622:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa6680 x1876702068054784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068054784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948643:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948646:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765723.948651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765723.948669:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa5180 x1876702068054912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068054912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948696:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068054912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765723.948708:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765723.948725:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948736:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa4700 x1876702068055040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948747:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068055040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948757:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948760:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765723.948765:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068055168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765723.948785:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948795:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa4380 x1876702068055168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948833:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068055168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948840:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948843:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948846:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068055168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765723.948851:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068055296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765723.948869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765723.948879:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765723/real 1789765723] req@ffff936010aa4e00 x1876702068055296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765723.948888:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068055296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765723.948896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765723.948898:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765723.948901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068055296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765724.330176:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765725.354294:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765726.378263:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765727.401947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765728.428606:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765729.066544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.066560:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765729.066565:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.066575:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.066586:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.066594:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765729.066649:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068055424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765729.066785:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068055424:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765729.075617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.075627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765729.075631:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.075641:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.075651:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.075658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.075677:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765729.075711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068055552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765729.075744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.075748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765729.075750:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.075756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.075760:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.075764:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.075768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765729.075780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068055680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765729.075788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.075791:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765729.075793:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.075822:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.075826:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.075830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.075834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765729.076138:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765729.076211:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765729.076519:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068055808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765729.076541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.076545:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765729.076547:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.076550:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.076555:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.076559:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.076565:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765729.076576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068055936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765729.076585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.076588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765729.076590:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.076593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.076596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.076599:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.076603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765729.076612:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068056064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765729.076620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765729.076623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765729.076624:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765729.076628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3196 00000100:00080000:1.0:1789765729.076631:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765729.076635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765729.076639:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765729.076648:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068056192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765729.077235:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765729.077281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765729.077318:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765729.077342:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765729.450301:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765730.474419:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765731.498877:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765732.138090:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765732.138127:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765732.138141:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765732.140995:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef7100 x1876702068055424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765754 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141027:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068055424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765754 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141039:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141043:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141047:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068055424:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765732.141273:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765732.141297:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765732.141304:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765732.141311:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765732.141436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef4700 x1876702068055808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141452:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068055808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141461:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765732.141475:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef5180 x1876702068055552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141492:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068055552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141499:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765732.141512:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef5c00 x1876702068055680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141524:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068055680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141531:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789765732.141598:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765732.141606:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765732.141655:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef4a80 x1876702068055936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068055936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068055936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765732.141689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef5880 x1876702068056064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141700:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068056064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765732.141717:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765729/real 1789765732] req@ffff936010ef4380 x1876702068056192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765732.141729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068056192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765732.141735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765732.141738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765732.141740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765732.522122:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765733.546639:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765734.186118:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.186129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765734.186133:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.186143:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.186154:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.186161:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765734.186198:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068056320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765734.186337:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068056320:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765734.186407:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765734.186439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.186443:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765734.186445:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.186450:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.186456:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.186460:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.186469:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765734.186484:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068056448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765734.186600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa7100 x1876702068056320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.186619:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068056320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.186628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.186632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.186636:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068056320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765734.186661:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765734.186682:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.186694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa5500 x1876702068056448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.186704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068056448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.186712:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.186715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.186718:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056448:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765734.190328:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.190338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765734.190342:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.190351:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.190362:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.190380:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.190395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765734.190429:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068056576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765734.190450:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.190453:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765734.190454:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.190458:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.190461:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.190465:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.190469:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765734.190482:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068056704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765734.190490:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.190493:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765734.190495:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.190498:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.190501:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.190504:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.190508:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765734.190518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068056832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765734.190525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.190528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765734.190530:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.190533:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.190537:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.190541:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.190545:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765734.190554:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068056960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765734.190562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765734.190565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765734.190567:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765734.190570:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3202 00000100:00080000:3.0:1789765734.190573:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765734.190576:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765734.190580:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765734.190590:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068057088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765734.190980:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765734.191042:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.191081:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa6a00 x1876702068056576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.191096:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068056576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.191105:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.191108:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.191123:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765734.191130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765734.191150:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.191161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa5500 x1876702068056704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.191172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068056704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.191179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.191182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.191184:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765734.191190:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765734.191208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.191325:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa7100 x1876702068056832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.191342:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068056832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.191350:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.191353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.191357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068056832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765734.191363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765734.191452:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.191465:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa4e00 x1876702068056960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.191475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068056960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.191482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.191486:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.191489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068056960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765734.191495:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765734.191514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765734.191524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765734/real 1789765734] req@ffff936010aa4380 x1876702068057088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765734.191534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068057088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765734.191541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765734.191544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765734.191547:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765734.571617:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765735.594088:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765736.618004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765737.644073:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765738.666118:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765739.308082:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308103:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765739.308107:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308117:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308127:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308135:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765739.308172:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068057216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765739.308203:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765739.308210:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308214:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308218:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308222:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308231:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765739.308243:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068057344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765739.308254:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308257:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765739.308259:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308263:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308266:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789765739.308269:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068057216:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765739.308272:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308281:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765739.308292:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068057472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765739.308301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765739.308306:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308309:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308313:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308317:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308322:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765739.308337:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068057600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789765739.308337:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789765739.308349:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308352:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765739.308354:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308358:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308362:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308365:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308370:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765739.308383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020dac000 x1876702068057216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765739.308384:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068057728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765739.308412:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308416:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765739.308418:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765739.308420:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068057216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765739.308425:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308432:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308438:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308447:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789765739.308448:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308459:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068057216:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765739.308462:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068057856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765739.308471:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057344:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789765739.308474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765739.308479:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765739.308481:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765739.308486:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3207 00000100:00080000:0.0:1789765739.308491:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765739.308495:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:3.0:1789765739.308500:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789765739.308503:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765739.308517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020dad880 x1876702068057344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765739.308518:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068057984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765739.308555:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068057344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308563:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308567:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308571:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765739.308686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765739.308715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765739.308726:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020dad500 x1876702068057472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765739.308737:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068057472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308745:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308748:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308751:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765739.308757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765739.308776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765739.308786:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020dae680 x1876702068057600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765739.308815:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068057600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308823:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308826:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308829:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765739.308834:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765739.308853:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765739.308867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020daed80 x1876702068057728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765739.308877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068057728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308888:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068057728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765739.308894:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765739.308911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765739.308920:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020dac700 x1876702068057856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765739.308930:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068057856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308937:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308940:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.308943:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765739.308948:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765739.308966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765739.308976:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765739/real 1789765739] req@ffff936020daf100 x1876702068057984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765739.308987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068057984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765739.308993:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765739.308997:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765739.309000:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068057984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765739.691947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765740.713995:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765741.738000:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765742.762397:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765743.788255:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765744.426072:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765744.426086:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426096:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426107:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426115:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765744.426149:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068058112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765744.426181:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426185:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426188:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426191:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426200:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765744.426210:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068058240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426220:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765744.426224:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426227:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426230:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426233:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426236:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765744.426244:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068058368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426253:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426256:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765744.426258:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426262:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426265:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426269:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765744.426280:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068058496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426292:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765744.426293:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426296:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426299:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426302:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426305:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765744.426312:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068058624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426321:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426325:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765744.426326:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426329:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426332:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426335:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426339:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765744.426347:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068058752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765744.426355:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765744.426358:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765744.426359:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765744.426362:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3212 00000100:00080000:3.0:1789765744.426365:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765744.426368:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765744.426371:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765744.426379:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068058880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765744.426457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068058112:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765744.426513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426552:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa4700 x1876702068058112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426568:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068058112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426577:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068058112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765744.426591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765744.426625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa5180 x1876702068058240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068058240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426657:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765744.426666:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765744.426684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa6680 x1876702068058368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068058368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426711:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426716:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765744.426720:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765744.426738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa7480 x1876702068058496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068058496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426766:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426773:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765744.426778:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765744.426795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa5f80 x1876702068058624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068058624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426845:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068058624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765744.426855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068058752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765744.426874:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa4a80 x1876702068058752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426896:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068058752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426904:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426910:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068058752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765744.426917:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068058880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765744.426934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765744.426945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765744/real 1789765744] req@ffff936010aa5880 x1876702068058880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765744.426956:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068058880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765744.426964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765744.426967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765744.426970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068058880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765744.810003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765745.834020:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765746.858721:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765747.882106:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765748.910385:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765749.547832:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.547841:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765749.547845:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.547855:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.547865:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.547872:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765749.547916:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068059008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.547944:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.547956:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765749.547958:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.547962:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.547965:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.547968:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.547977:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765749.547989:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068059136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.547998:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.548001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765749.548002:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.548006:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.548008:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.548011:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.548014:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765749.548023:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068059264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.548030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.548033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765749.548034:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.548037:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.548040:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.548043:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.548047:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765749.548054:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068059392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.548061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.548064:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765749.548065:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.548067:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.548070:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.548073:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.548077:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765749.548085:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068059520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.548092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.548094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765749.548096:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.548098:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.548101:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.548103:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.548107:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765749.548113:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068059648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765749.548125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765749.548128:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765749.548130:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765749.548133:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3217 00000100:00080000:1.0:1789765749.548135:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765749.548138:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765749.548141:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765749.548149:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068059776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765749.548239:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068059008:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765749.548291:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef4000 x1876702068059008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548340:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068059008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548349:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548353:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548356:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068059008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765749.548363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765749.548381:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548391:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef4380 x1876702068059136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548405:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068059136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548411:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548414:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548417:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765749.548421:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765749.548437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef5880 x1876702068059264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068059264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765749.548484:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765749.548502:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548513:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef4a80 x1876702068059392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548522:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068059392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548533:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548536:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548539:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765749.548544:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765749.548561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548571:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef5c00 x1876702068059520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068059520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548592:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068059520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765749.548608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068059648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765749.548628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef5180 x1876702068059648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548647:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068059648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548654:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068059648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765749.548665:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068059776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765749.548683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765749.548693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765749/real 1789765749] req@ffff936010ef4700 x1876702068059776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765749.548703:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068059776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765749.548709:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765749.548712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765749.548715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068059776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765749.934193:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765750.954029:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765751.978180:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765753.002158:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765754.027355:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765754.667824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.667834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765754.667838:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.667848:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.667859:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.667867:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765754.667903:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068059904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.667947:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.667950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765754.667951:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.667955:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.667958:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.667962:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.667972:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765754.667982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068060032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.667990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.667993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765754.667995:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.667998:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.668001:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.668004:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.668007:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765754.668024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068060160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.668032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.668035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765754.668036:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.668039:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.668042:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.668045:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.668049:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765754.668056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068060288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.668063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.668066:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765754.668068:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.668071:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.668073:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.668076:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.668079:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765754.668086:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068060416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.668093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.668096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765754.668097:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.668100:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.668102:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.668105:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.668109:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765754.668117:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068060544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765754.668124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765754.668127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765754.668128:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765754.668131:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3222 00000100:00080000:1.0:1789765754.668134:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765754.668137:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765754.668140:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765754.668148:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068060672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765754.668360:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068059904:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765754.668429:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef7100 x1876702068059904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668486:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068059904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668499:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068059904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765754.668510:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765754.668535:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668548:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef4e00 x1876702068060032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668559:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068060032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668567:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668585:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765754.668595:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765754.668618:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668630:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef6d80 x1876702068060160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668640:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068060160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668648:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668651:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765754.668660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765754.668680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef6300 x1876702068060288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668706:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068060288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668718:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765754.668734:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765754.668753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668763:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef7b80 x1876702068060416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668771:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068060416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668778:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668781:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668783:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765754.668788:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068060544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765754.668832:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668843:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef7480 x1876702068060544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668854:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068060544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668866:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668873:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068060544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765754.668878:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068060672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765754.668896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765754.668917:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765754/real 1789765754] req@ffff936010ef6680 x1876702068060672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765754.668928:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068060672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765754.668935:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765754.668937:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765754.668940:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068060672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765755.054145:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765756.074242:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765757.098643:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765758.122165:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765759.146539:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765759.785940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.785952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765759.785957:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.785967:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.785979:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.785987:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765759.786028:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068060800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786074:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765759.786080:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.786085:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786088:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786092:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786112:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765759.786125:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068060928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786134:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786138:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765759.786140:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.786143:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786147:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786151:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786155:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765759.786165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068061056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765759.786179:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.786182:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786185:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786188:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765759.786201:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068061184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786222:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765759.786223:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.786226:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786229:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786232:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786236:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765759.786244:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068061312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765759.786256:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765759.786259:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786261:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786264:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786268:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765759.786280:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068061440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765759.786290:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765759.786293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765759.786294:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765759.786300:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068060800:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765759.786301:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3227 00000100:00080000:3.0:1789765759.786306:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765759.786311:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765759.786316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765759.786327:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068061568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765759.786383:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.786500:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa5c00 x1876702068060800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.786519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068060800/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.786530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.786534:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.786538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068060800:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765759.786546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765759.786572:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.786585:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa7800 x1876702068060928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.786598:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068060928/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.786606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.786609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.786612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068060928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765759.786618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765759.786637:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.786648:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa4000 x1876702068061056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.786672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068061056/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.786681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.786683:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.786686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765759.786691:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765759.786865:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.786891:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa7b80 x1876702068061184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.786904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068061184/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.786912:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.786916:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.786919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765759.786925:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765759.786947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.786958:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa6300 x1876702068061312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.786969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068061312/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.786976:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.786980:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.786983:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765759.787017:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068061440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765759.787037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.787049:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa5880 x1876702068061440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.787067:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068061440/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.787075:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.787078:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.787081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068061440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765759.787087:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068061568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765759.787106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765759.787118:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765759/real 1789765759] req@ffff936010aa4a80 x1876702068061568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765759.787129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068061568/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765759.787137:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765759.787139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765759.787143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068061568:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765760.170005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765761.199362:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765762.219165:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765763.242068:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765764.266815:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765764.907095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765764.907110:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907120:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907130:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907138:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765764.907186:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068061696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765764.907220:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907223:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907228:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907232:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907242:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765764.907253:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068061824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765764.907267:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907270:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907273:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907276:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907279:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765764.907289:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068061952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907298:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907301:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765764.907302:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907305:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907309:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907312:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907316:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765764.907327:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068062080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907336:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765764.907340:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907343:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907346:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907349:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907354:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765764.907363:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068062208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907374:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765764.907376:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907379:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907382:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907385:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907389:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765764.907398:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068062336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765764.907406:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765764.907409:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765764.907411:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765764.907414:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3232 00000100:00080000:2.0:1789765764.907417:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765764.907420:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765764.907424:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765764.907433:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068062464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765764.907694:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068061696:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765764.907755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.907796:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061df480 x1876702068061696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.907865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068061696/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.907875:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.907879:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.907882:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068061696:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765764.907888:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765764.907912:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.907924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061dd500 x1876702068061824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.907935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068061824/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.907942:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.907945:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.907948:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765764.907954:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765764.907973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.907984:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061dc000 x1876702068061952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.908007:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068061952/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.908016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.908019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.908022:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068061952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765764.908028:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765764.908048:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.908059:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061dce00 x1876702068062080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.908070:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068062080/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.908078:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.908081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.908084:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765764.908089:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765764.908107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.908117:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061dd180 x1876702068062208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.908128:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068062208/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.908135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.908138:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.908141:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765764.908146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068062336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765764.908166:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.908364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061df800 x1876702068062336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.908378:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068062336/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.908386:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.908389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.908392:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068062336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765764.908398:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068062464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765764.908420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765764.908432:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765764/real 1789765764] req@ffff9360061dfb80 x1876702068062464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765764.908442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068062464/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765764.908448:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765764.908451:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765764.908454:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068062464:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765765.292573:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765766.314067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765767.338216:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765768.362486:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765769.386052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765770.026098:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026109:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765770.026112:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026122:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026133:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026141:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765770.026181:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068062592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026229:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765770.026233:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026237:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026243:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765770.026263:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068062720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026272:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026275:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765770.026277:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026281:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026284:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026287:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026292:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765770.026302:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068062848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026310:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026313:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765770.026314:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026318:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026321:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026325:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026330:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765770.026339:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068062976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026346:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026349:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765770.026350:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026353:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026356:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026359:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026362:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765770.026370:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068063104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026377:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026380:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765770.026381:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026384:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026387:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026390:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026394:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765770.026402:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068063232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765770.026410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765770.026413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765770.026414:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765770.026417:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3237 00000100:00080000:0.0:1789765770.026419:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765770.026422:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765770.026425:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765770.026432:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068063360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765770.026526:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068062592:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765770.026584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.026620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020daf480 x1876702068062592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.026635:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068062592/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.026643:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.026647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.026650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068062592:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765770.026657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765770.026677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.026689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020dafb80 x1876702068062720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.026699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068062720/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.026706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.026708:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.026711:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765770.026716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765770.026734:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.026744:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020dac380 x1876702068062848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.026755:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068062848/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.026762:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.026765:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.026767:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765770.026772:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765770.026793:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.026848:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020dad180 x1876702068062976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.026860:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068062976/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.026867:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.026871:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.026874:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068062976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765770.026880:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063104:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765770.026899:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.026910:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020dae300 x1876702068063104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.026921:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068063104/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.026928:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.026931:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.026934:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765770.027123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068063232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765770.027147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.027157:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020dace00 x1876702068063232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.027172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068063232/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.027179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.027182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.027185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068063232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765770.027189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068063360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765770.027204:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765770.027213:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765770/real 1789765770] req@ffff936020daea00 x1876702068063360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765770.027223:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068063360/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765770.027230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765770.027232:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765770.027235:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068063360:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765770.416301:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765771.441112:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765772.458393:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765773.482156:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765774.506773:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765775.146519:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.146529:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765775.146534:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765775.146543:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:3.0:1789765775.146554:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765775.146563:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765775.146598:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068063488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765775.146630:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.146633:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765775.146635:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765775.146639:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:3.0:1789765775.146643:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765775.146647:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765775.146656:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765775.146671:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068063616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765775.146672:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068063488:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765775.146688:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.146692:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765775.146694:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765775.146700:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:3.0:1789765775.146704:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765775.146707:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765775.146712:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765775.146725:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765775.146727:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068063744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765775.146738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.146742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765775.146744:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765775.146767:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00100000:0.0:1789765775.146772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa5f80 x1876702068063488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765775.146774:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765775.146782:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765775.146788:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765775.146823:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068063488/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765775.146831:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068063872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765775.146832:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765775.146839:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.146845:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068063488:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765775.146851:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.146855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765775.146857:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765775.146861:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:3.0:1789765775.146866:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765775.146867:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063616:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765775.146873:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765775.146878:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765775.146889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765775.146894:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068064000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765775.146901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa7480 x1876702068063616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765775.146911:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765775.146915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068063616/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765775.146917:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765775.146921:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765775.146926:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765775.146929:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:0.0:1789765775.146932:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.146937:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063616:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765775.146937:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765775.146944:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063744:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765775.146945:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765775.146953:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765775.146960:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765775.146970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa6680 x1876702068063744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765775.146978:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068064128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765775.146985:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068063744/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765775.146992:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765775.147000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765775.147003:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765775.147003:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765775.147012:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789765775.147013:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3242 00000100:00080000:3.0:1789765775.147020:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765775.147021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063744:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765775.147026:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765775.147033:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063872:192.168.203.137@tcp:8:kworker.0 00000008:00100000:3.0:1789765775.147033:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765775.147049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068064256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765775.147057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765775.147068:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa5180 x1876702068063872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765775.147077:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068063872/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765775.147083:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765775.147085:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.147087:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068063872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765775.147098:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765775.147108:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765775.147114:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa4700 x1876702068064000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765775.147122:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068064000/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765775.147129:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765775.147132:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.147135:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765775.147140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068064128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765775.147158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765775.147169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa4380 x1876702068064128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765775.147178:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068064128/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765775.147184:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765775.147186:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.147190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068064128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765775.147194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068064256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765775.147210:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765775.147219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765775/real 1789765775] req@ffff936010aa4e00 x1876702068064256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765775.147229:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068064256/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765775.147236:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765775.147239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765775.147242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068064256:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765775.534508:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765776.554622:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765777.580879:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765778.602989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765779.626356:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765780.266052:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765780.266066:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266076:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266086:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266093:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765780.266129:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068064384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765780.266165:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266170:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266173:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266177:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266185:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765780.266196:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068064512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765780.266209:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266211:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266214:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266217:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266221:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765780.266230:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068064640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765780.266242:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266245:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266249:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266252:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266256:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765780.266264:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068064768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765780.266276:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266279:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266282:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266286:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266290:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765780.266301:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068064896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765780.266313:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266316:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266319:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266323:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266327:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765780.266336:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068065024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765780.266344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765780.266347:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765780.266348:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765780.266352:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3248 00000100:00080000:1.0:1789765780.266355:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765780.266358:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765780.266362:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765780.266370:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068065152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765780.266457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068064384:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765780.266522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef6a00 x1876702068064384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266586:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068064384/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266599:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068064384:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765780.266610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765780.266631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef5f80 x1876702068064512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266654:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068064512/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266668:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765780.266674:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765780.266706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266718:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef5500 x1876702068064640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068064640/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765780.266758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064768:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765780.266778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef7800 x1876702068064768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068064768/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266847:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266849:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266852:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765780.266858:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765780.266883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266895:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef6680 x1876702068064896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068064896/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266913:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266916:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068064896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765780.266924:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765780.266944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.266955:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef7480 x1876702068065024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.266966:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068065024/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.266979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.266982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.266986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765780.266992:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765780.267011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765780.267021:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765780/real 1789765780] req@ffff936010ef7b80 x1876702068065152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765780.267032:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068065152/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765780.267039:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765780.267042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765780.267045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065152:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765780.650019:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765781.674475:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765782.699663:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765783.722010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765784.746057:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765785.389383:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389392:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765785.389396:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389405:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389415:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389422:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765785.389455:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068065280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389487:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765785.389493:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389496:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389499:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389502:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389511:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765785.389520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068065408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765785.389533:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389539:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389542:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389545:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765785.389552:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068065536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389559:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389562:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765785.389563:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389566:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389568:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389571:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389575:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765785.389583:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068065664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389591:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765785.389595:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389598:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389600:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389603:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389606:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765785.389612:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068065792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765785.389624:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389627:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389629:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389632:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389635:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765785.389641:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068065920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765785.389648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765785.389650:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765785.389652:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765785.389655:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3253 00000100:00080000:0.0:1789765785.389657:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765785.389669:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765785.389673:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765785.389681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068066048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765785.389863:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068065280:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765785.389924:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.389961:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020dadf80 x1876702068065280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.389977:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068065280/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.389986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.389990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.389994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068065280:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765785.390001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765785.390022:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390034:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020dadc00 x1876702068065408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390046:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068065408/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765785.390076:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765785.390095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390106:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020daf800 x1876702068065536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390116:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068065536/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390124:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390126:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390129:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765785.390134:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765785.390152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020daea00 x1876702068065664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390172:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068065664/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390184:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765785.390189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765785.390206:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390215:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020dace00 x1876702068065792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390224:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068065792/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390233:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390235:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068065792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765785.390240:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765785.390256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390265:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020dae300 x1876702068065920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390281:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068065920/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390288:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390293:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068065920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765785.390298:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765785.390314:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765785.390324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765785/real 1789765785] req@ffff936020dad180 x1876702068066048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765785.390334:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068066048/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765785.390341:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765785.390343:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765785.390346:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066048:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765785.772269:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765786.796326:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765787.818114:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765788.842088:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765789.871186:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765790.510041:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765790.510056:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510066:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510077:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510084:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765790.510121:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068066176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510154:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765790.510160:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510164:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510168:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510172:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510180:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765790.510193:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068066304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510202:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510204:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765790.510206:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510209:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510212:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510215:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510219:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765790.510229:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068066432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510237:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510239:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765790.510241:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510244:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510248:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510251:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510255:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765790.510265:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068066560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510279:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510282:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765790.510284:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510288:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510290:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510293:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510298:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765790.510307:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068066688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510315:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765790.510319:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510323:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510326:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510329:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510333:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765790.510342:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068066816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765790.510351:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765790.510354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765790.510355:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765790.510358:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3258 00000100:00080000:0.0:1789765790.510362:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765790.510365:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765790.510369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765790.510378:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068066944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765790.513440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068066176:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765790.513512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020dac380 x1876702068066176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513583:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068066176/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513601:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068066176:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765790.513615:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765790.513640:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020dafb80 x1876702068066304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068066304/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765790.513700:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765790.513721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020daf480 x1876702068066432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068066432/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513754:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513758:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513761:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765790.513767:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765790.513788:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513832:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020daf100 x1876702068066560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513843:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068066560/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513851:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513853:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765790.513863:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765790.513883:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513895:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020dac700 x1876702068066688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068066688/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513912:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513915:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068066688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765790.513925:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765790.513946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.513956:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020daed80 x1876702068066816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.513967:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068066816/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.513973:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.513976:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.513979:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765790.513984:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765790.514001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765790.514011:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765790/real 1789765790] req@ffff936020dae680 x1876702068066944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765790.514020:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068066944/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765790.514027:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765790.514030:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765790.514032:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068066944:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765790.903076:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765791.914005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765792.939446:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765793.962061:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765794.986070:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765795.625887:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.625898:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765795.625903:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.625913:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.625923:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.625930:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765795.625966:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068067072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.625996:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626000:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765795.626002:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.626006:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626010:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626013:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626021:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765795.626033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068067200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.626042:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626045:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765795.626047:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.626051:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626055:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626058:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626062:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765795.626072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068067328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.626080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626082:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765795.626084:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.626087:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626090:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626093:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626096:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765795.626106:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068067456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.626114:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626116:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765795.626117:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.626120:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626123:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626126:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626129:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765795.626137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068067584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.626156:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626159:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765795.626161:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765795.626164:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626167:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626170:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626174:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765795.626185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068067712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765795.626195:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765795.626198:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765795.626199:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789765795.626205:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068067072:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789765795.626206:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3263 00000100:00080000:0.0:1789765795.626212:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765795.626217:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765795.626222:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765795.626233:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068067840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765795.626314:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765795.626410:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765795.626437:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765795.626459:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765795.626488:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068067712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765795.626506:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068067840:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765796.011146:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765797.042245:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765798.058019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789765798.701138:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:0.0:1789765798.701181:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789765798.701195:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701231:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701239:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701247:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701254:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701260:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:0.0:1789765798.701266:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789765798.701339:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020dac700 x1876702068067712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068067712/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701373:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701378:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701382:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068067712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765798.701389:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020daf100 x1876702068067840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068067840/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701408:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701412:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701415:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068067840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765798.701421:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020dad500 x1876702068067072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068067072/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701444:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068067072:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765798.701450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020dad880 x1876702068067200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068067200/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701476:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701479:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765798.701484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020dac000 x1876702068067328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701494:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068067328/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701504:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765798.701512:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020dae680 x1876702068067456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701523:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068067456/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701533:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765798.701541:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765795/real 1789765798] req@ffff936020daed80 x1876702068067584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765798.701552:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068067584/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765798.701559:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765798.701562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765798.701565:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068067584:192.168.203.137@tcp:8:kworker.0 00010000:00010000:3.0:1789765799.083567:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765800.105996:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765800.747867:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.747877:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765800.747880:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.747899:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.747909:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.747915:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765800.747947:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068067968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.747977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.747981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765800.747983:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.747987:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.747991:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.747994:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748004:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765800.748015:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068068096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.748025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.748028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765800.748030:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.748034:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.748037:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.748041:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748045:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765800.748056:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068068224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.748064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.748067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765800.748069:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.748072:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.748075:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.748078:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748082:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765800.748092:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068068352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.748101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.748103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765800.748105:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.748107:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.748110:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.748113:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748117:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765800.748125:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068068480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.748133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.748137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765800.748138:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.748141:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.748144:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.748147:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748151:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765800.748160:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068068608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765800.748168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765800.748171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765800.748172:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765800.748175:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3268 00000100:00080000:2.0:1789765800.748178:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765800.748181:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765800.748185:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765800.748193:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068068736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765800.748256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068067968:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765800.748320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061de680 x1876702068067968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748387:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068067968/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748401:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068067968:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765800.748412:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765800.748431:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748442:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061dca80 x1876702068068096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068068096/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748463:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765800.748470:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765800.748485:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061dfb80 x1876702068068224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748508:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068068224/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748515:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748517:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748520:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765800.748524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765800.748539:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748554:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061df800 x1876702068068352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748563:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068068352/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748569:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748583:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748586:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068352:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765800.748591:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765800.748607:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748616:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061dd180 x1876702068068480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068068480/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748639:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748642:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748645:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765800.748650:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068068608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765800.748665:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748674:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061dce00 x1876702068068608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748684:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068068608/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748692:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748695:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748698:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068068608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765800.748702:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068068736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765800.748716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765800.748724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765800/real 1789765800] req@ffff9360061dc000 x1876702068068736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765800.748734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068068736/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765800.748740:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765800.748742:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765800.748746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068068736:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765801.131129:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765802.154573:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765803.177956:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765804.202167:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765805.226419:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765805.869580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869594:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765805.869600:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869624:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.869638:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.869647:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765805.869699:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068068864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.869738:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869742:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765805.869744:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869749:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.869753:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.869757:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.869769:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765805.869783:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068068992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.869794:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869814:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765805.869817:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869822:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.869826:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.869840:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.869847:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765805.869861:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068069120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.869874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869877:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765805.869880:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869886:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.869892:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789765805.869892:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068068864:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765805.869903:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.869911:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765805.869924:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068069248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.869934:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869937:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765805.869939:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869943:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.869947:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.869951:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.869955:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765805.869969:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068069376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.869977:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.869986:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:3.0:1789765805.869987:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765805.869990:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.869998:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.870003:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.870009:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.870014:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765805.870026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068069504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765805.870040:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765805.870046:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789765805.870046:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef6300 x1876702068068864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765805.870052:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765805.870065:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3273 00000100:00080000:1.0:1789765805.870073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765805.870081:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765805.870087:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765805.870091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068068864/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765805.870105:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068069632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765805.870106:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870121:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870134:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068068864:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765805.870146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765805.870173:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef6d80 x1876702068068992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870197:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068068992/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870204:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870207:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870210:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068068992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765805.870241:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765805.870262:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870272:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef4e00 x1876702068069120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870283:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068069120/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870292:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765805.870304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765805.870323:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870332:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef7100 x1876702068069248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068069248/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870354:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765805.870363:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765805.870394:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870406:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef4700 x1876702068069376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068069376/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870427:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870431:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870435:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765805.870441:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068069504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765805.870459:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870470:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef5180 x1876702068069504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870481:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068069504/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870488:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870492:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870494:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068069504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765805.870500:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068069632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765805.870518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765805.870550:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765805/real 1789765805] req@ffff936010ef5c00 x1876702068069632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765805.870562:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068069632/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765805.870574:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765805.870577:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765805.870580:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068069632:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765806.250617:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765807.275780:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765808.298992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765809.323310:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765810.348718:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765810.990998:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991008:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765810.991012:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991021:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991031:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991038:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765810.991072:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068069760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991104:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991108:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765810.991109:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991113:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991117:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991120:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991128:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765810.991139:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068069888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991147:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991150:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765810.991152:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991155:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991158:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991161:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991165:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765810.991174:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068070016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991181:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991184:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765810.991186:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991189:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991191:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991194:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991198:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765810.991205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068070144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991212:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991215:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765810.991216:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991219:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991222:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991225:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765810.991236:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068070272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991244:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991246:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765810.991248:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991251:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991254:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991256:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991260:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765810.991268:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068070400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765810.991275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765810.991278:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765810.991279:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765810.991282:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3278 00000100:00080000:0.0:1789765810.991285:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765810.991288:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765810.991292:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765810.991299:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068070528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765810.991424:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068069760:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765810.991641:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.991749:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020daf480 x1876702068069760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.991767:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068069760/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.991776:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.991780:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.991784:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068069760:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765810.991790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765810.991848:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.991860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020dafb80 x1876702068069888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.991872:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068069888/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.991885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.991889:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.991892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068069888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765810.991897:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765810.991917:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.991928:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020dac380 x1876702068070016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.991938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068070016/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.991945:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.991947:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.991950:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765810.991955:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765810.991973:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.991983:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020dad180 x1876702068070144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.991994:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068070144/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.992001:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.992005:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.992008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765810.992013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765810.992031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.992041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020dae300 x1876702068070272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.992051:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068070272/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.992057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.992060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.992063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765810.992068:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068070400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765810.992084:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.992094:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020dace00 x1876702068070400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.992103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068070400/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.992110:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.992113:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.992116:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068070400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765810.992122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068070528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765810.992138:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765810.992148:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765810/real 1789765810] req@ffff936020daea00 x1876702068070528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765810.992159:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068070528/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765810.992167:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765810.992170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765810.992173:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068070528:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765811.371953:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765812.394037:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765813.418136:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765814.442250:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765815.466168:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765816.106423:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.106432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765816.106596:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.106607:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.106618:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.106626:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765816.106664:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068070656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.106694:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.106698:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765816.106699:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.106703:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.106706:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.106710:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.106720:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765816.106731:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068070784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.106740:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.106743:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765816.106744:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.106748:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.106751:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.106754:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.106757:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765816.106962:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068070912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.106973:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.106976:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765816.106978:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.106982:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.106987:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.106990:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.106996:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765816.107008:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068071040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.107016:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.107019:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765816.107020:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.107024:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.107027:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.107030:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.107034:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765816.107045:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068071168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.107053:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.107056:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765816.107057:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.107061:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.107064:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.107067:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.107071:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765816.107080:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068071296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765816.107089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765816.107091:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765816.107093:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765816.107096:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3283 00000100:00080000:1.0:1789765816.107171:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765816.107175:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765816.107180:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765816.107190:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068071424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789765816.107522:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068070656:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789765816.107579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107617:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef4a80 x1876702068070656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107635:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068070656/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107646:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107653:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068070656:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789765816.107661:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765816.107683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef5880 x1876702068070784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068070784/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107719:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107722:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765816.107728:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765816.107745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107755:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef4380 x1876702068070912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068070912/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107774:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107777:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068070912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765816.107786:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765816.107826:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107838:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef4000 x1876702068071040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107849:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068071040/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107862:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107866:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765816.107875:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789765816.107895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef5c00 x1876702068071168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107918:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068071168/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107926:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107929:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107932:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765816.107938:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068071296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765816.107956:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.107966:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef5180 x1876702068071296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.107977:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068071296/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.107985:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.107988:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.107992:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068071296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765816.107998:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068071424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789765816.108016:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765816.108025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765816/real 1789765816] req@ffff936010ef4700 x1876702068071424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765816.108036:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068071424/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765816.108043:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765816.108046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789765816.108049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068071424:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765816.494078:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765817.514104:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765818.539434:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765819.568403:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765820.586008:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765821.228522:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765821.228532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765821.228536:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765821.228545:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:3.0:1789765821.228555:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765821.228562:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765821.228597:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068071552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765821.228680:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068071552:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765821.228957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765821.229015:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765821.229019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765821.229021:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765821.229025:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:3.0:1789765821.229029:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765821.229033:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765821.229042:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765821.229066:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068071680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765821.232643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff936010aa7100 x1876702068071552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.232683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068071552/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.232694:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.232699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.232703:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068071552:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765821.232907:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765821.232961:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.232978:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff936010aa5500 x1876702068071680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.232990:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068071680/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.233003:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.233008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.233011:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071680:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765821.238350:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765821.238360:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765821.238364:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765821.238376:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:3.0:1789765821.238397:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765821.238405:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765821.238629:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765821.238665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068071808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765821.238680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765821.238684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765821.238686:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765821.238690:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:2.0:1789765821.238695:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765821.238700:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765821.238704:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765821.238715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068071936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765821.238725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765821.238727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765821.238729:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765821.238733:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:2.0:1789765821.238736:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765821.238740:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765821.238744:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765821.238754:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068072064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765821.238762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765821.238765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765821.238767:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765821.238770:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:2.0:1789765821.238773:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765821.238776:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765821.238780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765821.238788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068072192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765821.238796:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765821.238836:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765821.238838:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765821.238841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3289 00000100:00080000:2.0:1789765821.238844:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765821.238848:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765821.238852:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765821.238862:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068072320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765821.238932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765821.238993:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.239019:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff9360061dd500 x1876702068071808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.239036:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068071808/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.239046:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.239049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.239053:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765821.239060:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765821.239080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.239092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff9360061df480 x1876702068071936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.239103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068071936/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.239111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.239114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.239117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068071936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765821.239122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765821.239139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.239150:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff9360061df100 x1876702068072064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.239161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068072064/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.239169:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.239172:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.239174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765821.239180:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068072192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765821.239198:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.239208:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff9360061dd880 x1876702068072192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.239218:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068072192/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.239226:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.239229:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.239232:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068072192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765821.239238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068072320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765821.239254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765821.239263:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765821/real 1789765821] req@ffff9360061ddc00 x1876702068072320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765821.239274:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068072320/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765821.239287:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765821.239290:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765821.239293:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068072320:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765821.610252:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765822.640027:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765823.657999:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765824.683561:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765825.706272:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765826.345924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.345931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765826.345935:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.345945:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.345956:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.345962:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765826.345997:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068072448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765826.346033:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346037:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346040:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.346044:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346053:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765826.346064:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068072576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346076:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765826.346078:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346081:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346084:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.346087:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765826.346098:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068072704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765826.346109:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346112:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346115:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.346118:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346122:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765826.346131:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068072832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765826.346144:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346148:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346151:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.346154:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346158:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765826.346161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068072448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789765826.346168:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068072960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765826.346182:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346185:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346188:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765826.346192:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346195:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765826.346204:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068073088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765826.346212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765826.346215:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765826.346217:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765826.346220:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3294 00000100:00080000:2.0:1789765826.346224:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789765826.346226:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765826.346230:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765826.346235:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765826.346246:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068073216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765826.346272:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061dc700 x1876702068072448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346290:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068072448/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346305:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346308:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068072448:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765826.346315:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765826.346338:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346351:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061ddf80 x1876702068072576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068072576/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346391:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346394:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765826.346400:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765826.346421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346432:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061dc380 x1876702068072704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068072704/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346460:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765826.346472:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765826.346490:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346499:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061de300 x1876702068072832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346509:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068072832/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346516:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346519:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346522:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765826.346546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072960:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765826.346565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346575:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061ddc00 x1876702068072960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346585:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068072960/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346592:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346595:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346598:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068072960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765826.346604:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765826.346623:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061dd880 x1876702068073088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346643:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068073088/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765826.346660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765826.346677:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765826.346687:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765826/real 1789765826] req@ffff9360061df100 x1876702068073216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765826.346705:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068073216/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765826.346712:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765826.346715:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765826.346718:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073216:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765826.733196:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765827.753982:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765828.778525:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765829.803350:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765830.830983:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765831.468599:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765831.468611:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468661:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468671:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468678:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765831.468712:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068073344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468742:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468746:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765831.468747:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468751:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468754:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468758:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468766:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765831.468777:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068073472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468786:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468789:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765831.468791:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468794:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468816:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468820:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468824:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765831.468834:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068073600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468843:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468846:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765831.468847:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468850:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468853:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468857:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468861:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765831.468868:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068073728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468876:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468879:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765831.468880:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468882:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468884:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468887:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468891:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765831.468898:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068073856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468906:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468908:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765831.468910:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468913:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468915:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468919:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468922:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765831.468930:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068073984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765831.468951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765831.468954:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765831.468956:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765831.468959:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3299 00000100:00080000:3.0:1789765831.468962:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765831.468965:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765831.468969:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765831.468977:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068074112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765831.469012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068073344:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765831.469066:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa6a00 x1876702068073344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469231:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068073344/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469243:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469250:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068073344:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765831.469258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765831.469282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469294:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa5500 x1876702068073472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068073472/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469321:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469329:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765831.469335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765831.469368:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa7100 x1876702068073600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469396:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068073600/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765831.469416:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765831.469436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa4e00 x1876702068073728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469523:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068073728/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469531:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765831.469545:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073856:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765831.469565:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa4380 x1876702068073856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469587:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068073856/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469595:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469601:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068073856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765831.469606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765831.469625:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469635:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa4700 x1876702068073984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068073984/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469652:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469655:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469658:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068073984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765831.469663:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068074112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765831.469681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765831.469691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765831/real 1789765831] req@ffff936010aa5180 x1876702068074112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765831.469701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068074112/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765831.469709:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765831.469712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765831.469715:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068074112:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765831.850155:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765832.874460:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765833.898179:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765834.922728:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765835.946153:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765836.588508:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.588518:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765836.588522:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.588629:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.588640:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.588648:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765836.588685:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068074240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765836.589113:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068074240:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765836.589495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765836.589701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.589706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765836.589709:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.589714:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.589719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.589724:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.589841:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765836.589861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068074368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765836.595533:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061df480 x1876702068074240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.595564:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068074240/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.595575:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.595579:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.595584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068074240:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765836.595615:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765836.595673:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.595691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061dd500 x1876702068074368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.595711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068074368/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.595720:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.595723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.595727:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074368:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789765836.595881:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.595887:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765836.595890:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.595896:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.595902:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.595909:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.595918:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765836.595943:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068074496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765836.595957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.595960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765836.595962:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.595965:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.595968:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.595972:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.595977:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765836.595986:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068074624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765836.595995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.595998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765836.596000:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.596003:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.596007:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.596010:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.596014:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765836.596024:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068074752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765836.596033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.596037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765836.596038:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.596042:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.596045:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.596049:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.596053:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765836.596062:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068074880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765836.596071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765836.596075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765836.596076:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765836.596080:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3304 00000100:00080000:2.0:1789765836.596083:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765836.596086:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765836.596090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765836.596099:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068075008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765836.596169:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765836.596195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.596209:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061dc000 x1876702068074496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.596222:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068074496/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.596230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.596234:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.596237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765836.596243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765836.596263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.596273:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061dce00 x1876702068074624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.596284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068074624/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.596298:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.596302:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.596305:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765836.596325:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074752:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765836.596345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.596357:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061dd180 x1876702068074752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.596369:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068074752/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.596378:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.596381:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.596384:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068074752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765836.596390:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068074880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765836.596408:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.596419:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061df800 x1876702068074880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.596431:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068074880/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.596438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.596441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.596445:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068074880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765836.596450:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765836.596469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765836.596479:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765836/real 1789765836] req@ffff9360061dfb80 x1876702068075008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765836.596491:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068075008/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765836.596499:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765836.596501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765836.596504:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075008:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765836.970746:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765837.995117:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765839.023360:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765840.043239:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765841.067645:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765841.706008:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706018:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765841.706022:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706032:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706042:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706050:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765841.706090:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068075136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765841.706167:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068075136:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765841.706239:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765841.706284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765841.706292:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706296:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706301:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706305:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706315:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765841.706330:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068075264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765841.706462:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa6680 x1876702068075136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.706482:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068075136/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.706492:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.706497:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.706501:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068075136:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765841.706524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765841.706549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.706563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa7480 x1876702068075264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.706575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068075264/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.706583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.706587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.706590:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075264:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765841.706648:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706652:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765841.706654:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706658:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706663:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706667:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706673:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765841.706687:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068075392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765841.706696:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706699:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765841.706701:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706704:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706708:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706711:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706716:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765841.706727:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068075520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765841.706735:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706738:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765841.706740:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706744:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706747:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706750:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706755:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765841.706765:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068075648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765841.706774:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706777:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765841.706778:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706781:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706784:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706788:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706792:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765841.706832:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068075776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765841.706841:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765841.706845:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765841.706847:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765841.706851:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3309 00000100:00080000:3.0:1789765841.706854:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765841.706858:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765841.706864:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765841.706874:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068075904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765841.706941:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765841.706966:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.706980:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa5f80 x1876702068075392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.706992:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068075392/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.707001:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.707004:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.707007:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765841.707013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765841.707033:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.707045:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa4a80 x1876702068075520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.707064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068075520/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.707072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.707075:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.707079:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765841.707085:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765841.707104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.707115:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa5880 x1876702068075648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.707126:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068075648/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.707133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.707136:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.707139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068075648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765841.707145:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765841.707163:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.708725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa6300 x1876702068075776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.708754:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068075776/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.708772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.708776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.708780:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765841.708790:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765841.708847:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765841.708867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765841/real 1789765841] req@ffff936010aa7b80 x1876702068075904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765841.708879:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068075904/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765841.708887:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765841.708890:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765841.708893:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068075904:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765842.090115:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765843.115600:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765844.138170:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765845.162022:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765846.191016:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765846.826291:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826301:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765846.826305:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826313:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826323:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826329:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765846.826362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068076032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826390:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826393:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765846.826395:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826399:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826402:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826405:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826413:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765846.826424:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068076160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826432:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826436:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765846.826438:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826485:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826488:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826492:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826497:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765846.826506:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068076288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826518:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765846.826520:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826523:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826525:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826529:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826533:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765846.826540:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068076416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826548:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826551:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765846.826552:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826555:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826557:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826560:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826564:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765846.826573:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068076544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826580:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826583:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765846.826585:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826588:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826591:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826593:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826597:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765846.826604:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068076672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765846.826611:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765846.826614:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765846.826616:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765846.826619:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3314 00000100:00080000:1.0:1789765846.826621:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765846.826624:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765846.826627:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765846.826634:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068076800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765846.826713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068076032:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765846.826770:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.826843:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef7100 x1876702068076032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.826859:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068076032/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.826874:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.826877:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.826881:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068076032:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765846.826888:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765846.826907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.826918:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef4e00 x1876702068076160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.826929:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068076160/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.826936:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.826939:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.826942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765846.826947:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765846.826962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.826970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef6d80 x1876702068076288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.826980:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068076288/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.826993:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.826996:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.826999:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765846.827005:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076416:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765846.827020:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.827029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef6300 x1876702068076416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.827039:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068076416/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.827047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.827049:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.827052:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765846.827057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765846.827071:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.827079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef7b80 x1876702068076544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.827089:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068076544/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.827097:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.827100:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.827103:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068076544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765846.827108:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068076672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765846.827121:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.827129:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef7480 x1876702068076672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.827141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068076672/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.827148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.827150:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.827153:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068076672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765846.827158:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068076800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765846.827172:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765846.827180:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765846/real 1789765846] req@ffff936010ef6680 x1876702068076800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765846.827190:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068076800/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765846.827198:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765846.827200:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765846.827203:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068076800:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765847.210122:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765848.234558:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765849.263835:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765850.282248:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765851.306094:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765851.946237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765851.946252:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946262:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946272:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946280:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765851.946317:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068076928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946409:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946413:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765851.946415:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946419:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946422:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946426:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946434:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765851.946446:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068077056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946455:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765851.946459:0:37:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946463:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946466:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946469:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946473:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765851.946485:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068077184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765851.946498:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946502:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946506:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946509:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946513:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765851.946522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068077312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946533:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765851.946535:0:37:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946538:0:37:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946541:0:37:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946544:0:37:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946548:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765851.946556:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068077440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765851.946568:0:37:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946570:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946573:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946576:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946580:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765851.946588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068077568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765851.946602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765851.946605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765851.946606:0:37:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765851.946610:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3319 00000100:00080000:2.0:1789765851.946613:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765851.946616:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765851.946620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765851.946629:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068077696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765851.947048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068076928:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765851.947158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.947206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061dca80 x1876702068076928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.947230:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068076928/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.947240:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.947243:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.947256:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068076928:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765851.947263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765851.947286:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.947298:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061de680 x1876702068077056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.947309:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068077056/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.947317:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.947320:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.947323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765851.947329:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765851.948302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.948652:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061dfb80 x1876702068077184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.948668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068077184/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.948683:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.948686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.948689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765851.948696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077312:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765851.948720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.949168:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061df800 x1876702068077312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.949183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068077312/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.949191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.949203:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.949207:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077312:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765851.949214:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765851.949237:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.954313:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061dd180 x1876702068077440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.954345:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068077440/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.954357:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.954361:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.954366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765851.954378:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068077568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765851.954436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.954455:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061dce00 x1876702068077568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.954465:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068077568/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.954473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.954476:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.954479:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068077568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765851.954485:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068077696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765851.954506:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765851.954517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765851/real 1789765851] req@ffff9360061dc000 x1876702068077696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765851.954529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068077696/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765851.954536:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765851.954539:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765851.954542:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068077696:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765852.331206:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765853.357345:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765854.380477:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765855.403669:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765856.427238:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765857.068343:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765857.068358:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068368:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068379:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068387:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765857.068424:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068077824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068455:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765857.068457:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068461:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068464:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068468:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068476:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765857.068487:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068077952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068494:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068498:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765857.068500:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068503:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068506:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068509:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068513:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765857.068522:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068078080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068530:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068533:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765857.068534:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068538:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068541:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068544:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068547:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765857.068556:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068078208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068563:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765857.068567:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068570:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068572:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068575:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068579:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765857.068588:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068078336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068596:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068599:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765857.068601:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068604:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068607:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068611:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068615:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765857.068624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068078464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765857.068637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765857.068640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765857.068642:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765857.068646:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3324 00000100:00080000:1.0:1789765857.068649:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765857.068652:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765857.068656:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765857.068665:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068078592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765857.068754:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068077824:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765857.068842:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.068877:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef7800 x1876702068077824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.068891:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068077824/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.068900:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.068903:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.068906:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068077824:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765857.068913:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765857.068933:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.068944:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef5500 x1876702068077952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.068954:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068077952/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.068961:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.068964:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.068967:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068077952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765857.068973:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765857.068991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.069003:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef5f80 x1876702068078080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.069014:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068078080/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.069022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.069025:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.069028:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765857.069034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078208:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765857.069052:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.069063:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef6a00 x1876702068078208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.069074:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068078208/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.069082:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.069085:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.069088:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078208:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765857.069093:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765857.069111:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.069122:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef6680 x1876702068078336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.069133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068078336/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.069141:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.069144:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.069147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765857.069152:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068078464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765857.069183:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.069194:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef7480 x1876702068078464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.069205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068078464/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.069211:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.069213:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.069216:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068078464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765857.069231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068078592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765857.069252:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765857.069262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765857/real 1789765857] req@ffff936010ef7b80 x1876702068078592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765857.069273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068078592/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765857.069280:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765857.069282:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765857.069286:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068078592:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765857.450053:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765858.475308:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765859.501456:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765860.523109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765861.546784:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765862.185909:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.185918:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765862.185922:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.185932:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.185942:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.185948:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765862.185980:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068078720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765862.186048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068078720:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765862.186159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.186163:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765862.186165:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.186169:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.186173:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.186177:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.186186:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765862.186208:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068078848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765862.189723:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078848:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765862.193984:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.193996:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765862.194000:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.194010:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.194020:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.194027:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.194039:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765862.194074:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068078976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765862.194090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.194094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765862.194096:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.194099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.194103:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.194106:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.194110:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765862.194118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068079104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765862.194139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.194142:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765862.194144:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.194147:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.194151:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.194154:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.194158:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765862.194167:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068079232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765862.194175:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.194177:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765862.194179:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.194182:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.194185:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.194188:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.194192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765862.194201:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068079360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765862.194209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765862.194212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765862.194213:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765862.194217:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3330 00000100:00080000:1.0:1789765862.194219:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765862.194222:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765862.194225:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765862.194234:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068079488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765862.194341:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765862.194407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765862.194542:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765862.194568:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068079360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765862.194589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068079488:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765862.572328:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765863.595204:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765864.618144:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765865.258090:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765865.258144:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765865.258161:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258204:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258213:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258221:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258229:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258236:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765865.258244:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765865.258421:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef4e00 x1876702068078976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258447:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068078976/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258457:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765865.258472:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef7100 x1876702068079104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258484:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068079104/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258491:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258495:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258497:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079104:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765865.258503:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef4700 x1876702068079232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068079232/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765865.258532:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef5180 x1876702068079360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068079360/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258550:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258553:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258556:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068079360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765865.258562:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef5c00 x1876702068079488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258573:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068079488/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068079488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765865.258591:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef6d80 x1876702068078848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258601:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068078848/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258607:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258610:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258613:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068078848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765865.258619:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765862/real 1789765865] req@ffff936010ef6300 x1876702068078720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765865.258630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068078720/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765865.258636:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765865.258639:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765865.258642:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068078720:192.168.203.137@tcp:250:kworker.0 00010000:00010000:0.0:1789765865.643269:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765866.667206:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765867.306094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306105:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765867.306109:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306119:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306142:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306150:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765867.306185:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068079616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765867.306348:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306353:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306357:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306361:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.306369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765867.306382:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068079744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306392:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306395:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765867.306397:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306400:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306404:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306408:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.306412:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765867.306422:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068079872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306432:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765867.306434:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306437:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306440:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306444:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.306448:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765867.306456:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068080000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306465:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306468:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765867.306469:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306472:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306474:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306477:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.306481:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765867.306491:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068080128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306498:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306502:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765867.306504:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306507:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306510:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.306513:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.306517:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765867.306527:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068080256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765867.306535:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765867.306538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765867.306539:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765867.306542:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3335 00000100:00080000:1.0:1789765867.306545:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765867.307768:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765867.307778:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765867.308151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068080384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765867.308252:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068079616:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765867.308311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.308355:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef4000 x1876702068079616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.308373:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068079616/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.308384:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.308387:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.308391:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068079616:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765867.308399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765867.308421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.308433:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef4380 x1876702068079744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.308444:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068079744/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.308451:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.308454:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.308457:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765867.308464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765867.308483:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.308494:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef5880 x1876702068079872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.308512:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068079872/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.308519:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.308522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.308537:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068079872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765867.308544:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080000:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765867.311418:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.311450:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef4a80 x1876702068080000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.311469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068080000/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.311481:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.311485:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.311489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080000:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765867.311499:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765867.311520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.313679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef6300 x1876702068080128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.313701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068080128/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.313719:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.313723:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.313727:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080128:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765867.313737:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068080256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765867.313783:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.313794:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef6d80 x1876702068080256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.314313:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068080256/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.314323:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.314327:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.314330:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068080256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765867.314338:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068080384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765867.314364:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765867.314377:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765867/real 1789765867] req@ffff936010ef5c00 x1876702068080384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765867.314389:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068080384/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765867.314397:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765867.314400:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765867.314404:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068080384:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765867.695010:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765868.714531:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765869.739342:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765870.763841:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765871.786021:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765872.425914:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.425923:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765872.425927:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.425936:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.425946:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.425953:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765872.425987:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068080512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426015:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426018:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765872.426019:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426023:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426026:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426029:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426038:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765872.426060:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068080640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426080:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765872.426081:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426084:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426087:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426091:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426094:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765872.426104:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068080768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426114:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765872.426116:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426118:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426121:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426125:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426129:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765872.426137:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068080896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426145:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765872.426148:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426151:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426154:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426157:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426160:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765872.426169:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068081024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765872.426180:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426183:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426186:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426188:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426192:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765872.426200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068081152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765872.426214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765872.426217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765872.426219:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765872.426222:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3340 00000100:00080000:1.0:1789765872.426225:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765872.426228:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765872.426232:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765872.426241:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068081280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765872.426331:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068080512:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765872.426388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.426437:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef5180 x1876702068080512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.426453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068080512/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.426462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.426465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.426469:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068080512:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765872.426475:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765872.426507:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.426518:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef4700 x1876702068080640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.426529:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068080640/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.426535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.426538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.426541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765872.426546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765872.426563:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.426587:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef7100 x1876702068080768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.426599:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068080768/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.426605:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.426608:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.426611:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765872.426615:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080896:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765872.426640:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.426651:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef4e00 x1876702068080896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.426661:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068080896/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.426667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.426670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.426672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068080896:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765872.426678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765872.426962:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.426977:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef7b80 x1876702068081024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.426989:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068081024/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.426997:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.427000:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.427003:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765872.427008:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068081152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765872.427030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.427041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef7480 x1876702068081152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.427064:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068081152/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.427072:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.427075:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.427078:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068081152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765872.427084:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068081280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765872.427103:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765872.427112:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765872/real 1789765872] req@ffff936010ef6680 x1876702068081280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765872.427122:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068081280/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765872.427133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765872.427136:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765872.427139:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068081280:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765872.813171:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765873.834951:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765874.862766:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765875.886655:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765876.906261:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765877.545961:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.545970:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765877.545974:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.545984:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.545994:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546001:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765877.546046:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068081408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546080:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546084:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765877.546086:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546090:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546098:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546106:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765877.546118:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068081536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546127:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765877.546132:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546136:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546139:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546142:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546146:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765877.546156:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068081664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546165:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546168:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765877.546170:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546173:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546177:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546180:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546185:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765877.546196:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068081792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546205:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546208:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765877.546210:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546213:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546217:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546220:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546224:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765877.546232:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068081920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546243:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765877.546244:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546247:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546250:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546254:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546257:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765877.546266:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068082048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765877.546280:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765877.546283:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765877.546284:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765877.546288:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3345 00000100:00080000:3.0:1789765877.546291:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765877.546294:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765877.546298:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765877.546307:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068082176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765877.546389:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068081408:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765877.546450:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546497:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa4000 x1876702068081408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068081408/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546529:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068081408:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765877.546537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765877.546558:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa7800 x1876702068081536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068081536/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546596:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546599:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765877.546608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765877.546626:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546637:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa5c00 x1876702068081664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546648:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068081664/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546658:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546660:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765877.546665:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081792:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765877.546684:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546694:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa7b80 x1876702068081792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546706:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068081792/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546714:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546717:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081792:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765877.546726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765877.546744:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546754:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa6300 x1876702068081920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546765:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068081920/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546773:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546776:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546779:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068081920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765877.546785:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765877.546835:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546847:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa5880 x1876702068082048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546859:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068082048/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546867:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546872:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765877.546878:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765877.546896:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765877.546906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765877/real 1789765877] req@ffff936010aa4a80 x1876702068082176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765877.546923:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068082176/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765877.546930:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765877.546933:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765877.546935:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082176:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765877.930056:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765878.955369:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765879.978333:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765881.003495:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765882.026127:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765882.669285:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669296:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765882.669301:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669310:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669320:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669328:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765882.669364:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068082304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765882.669394:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669398:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765882.669399:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669403:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669407:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669410:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669421:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765882.669433:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068082432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765882.669443:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765882.669448:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669451:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669455:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669458:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669462:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765882.669473:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068082560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765882.669480:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765882.669485:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669488:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669491:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669494:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669498:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765882.669507:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068082688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765882.669508:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068082304:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765882.669515:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669517:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765882.669519:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669522:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669524:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669527:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669530:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765882.669538:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068082816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765882.669545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669548:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765882.669549:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669555:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669558:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789765882.669560:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669562:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765882.669569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068082944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765882.669576:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765882.669579:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765882.669580:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765882.669583:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3350 00000100:00080000:1.0:1789765882.669586:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765882.669588:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765882.669592:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765882.669594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef6a00 x1876702068082304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765882.669600:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068083072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765882.669610:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068082304/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669630:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669638:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068082304:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765882.669645:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765882.669668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.669680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef5f80 x1876702068082432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.669698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068082432/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669707:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669710:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765882.669719:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765882.669737:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.669748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef5500 x1876702068082560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.669759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068082560/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669766:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669772:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765882.669795:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082688:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765882.669839:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.669851:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef7800 x1876702068082688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.669861:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068082688/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669872:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669875:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669877:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082688:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765882.669882:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765882.669898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.669906:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef6680 x1876702068082816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.669915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068082816/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669924:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068082816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765882.669932:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765882.669948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.669958:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef7480 x1876702068082944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.669969:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068082944/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.669986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.669990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.669993:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068082944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765882.669999:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765882.670018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765882.670028:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765882/real 1789765882] req@ffff936010ef7b80 x1876702068083072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765882.670038:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068083072/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765882.670045:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765882.670048:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765882.670061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083072:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765883.051362:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765884.076051:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765885.101376:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765886.125773:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765887.146062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765887.786053:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786062:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765887.786066:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786075:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786084:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786091:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765887.786126:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068083200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786162:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786165:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765887.786167:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786170:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786173:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786176:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786185:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765887.786200:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068083328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786210:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786214:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765887.786215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786219:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786222:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786225:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786229:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765887.786238:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068083456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786246:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786248:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765887.786249:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786253:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786256:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786259:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786262:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765887.786268:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068083584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786276:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786279:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765887.786280:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786283:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786285:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786288:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786292:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765887.786298:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068083712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786306:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786308:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765887.786310:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786323:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786326:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786329:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786332:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765887.786340:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068083840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765887.786348:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765887.786351:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765887.786352:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765887.786355:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3355 00000100:00080000:2.0:1789765887.786357:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765887.786360:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765887.786363:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765887.786369:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068083968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765887.786475:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068083200:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765887.786531:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786568:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061dd500 x1876702068083200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068083200/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786602:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068083200:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765887.786616:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765887.786634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786644:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061df480 x1876702068083328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068083328/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786663:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765887.786673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765887.786687:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786695:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061df100 x1876702068083456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068083456/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786711:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786714:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765887.786722:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083584:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765887.786735:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786743:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061dd880 x1876702068083584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786753:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068083584/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786760:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083584:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765887.786769:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765887.786782:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061ddc00 x1876702068083712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068083712/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786839:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068083712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765887.786855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765887.786872:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786881:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061de300 x1876702068083840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786890:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068083840/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786903:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786906:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786909:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765887.786913:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765887.786926:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765887.786945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765887/real 1789765887] req@ffff9360061dc380 x1876702068083968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765887.786956:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068083968/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765887.786964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765887.786967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765887.786969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068083968:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765888.171638:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765889.194425:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765890.218093:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765891.242613:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765892.266044:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765892.905969:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.905979:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765892.905983:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.905994:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906005:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906023:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765892.906062:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068084096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906092:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765892.906244:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068084096:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765892.906308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.906368:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061ddf80 x1876702068084096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.906385:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068084096/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.906401:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.906405:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.906409:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068084096:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765892.906562:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765892.906565:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906569:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906574:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906578:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906588:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765892.906603:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068084224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906624:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.906628:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765892.906630:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906634:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906637:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906641:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906645:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765892.906656:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068084352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906665:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.906668:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765892.906670:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906673:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906676:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906690:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906695:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765892.906750:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068084480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906759:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.906761:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765892.906763:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906766:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906769:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906773:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906777:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765892.906786:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068084608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906795:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.906818:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765892.906820:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906823:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906830:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906834:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906838:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765892.906849:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068084736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765892.906857:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765892.906860:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765892.906862:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765892.906865:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3360 00000100:00080000:2.0:1789765892.906869:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765892.906872:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765892.906875:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765892.906885:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068084864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765892.907383:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765892.907421:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907441:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061dc700 x1876702068084224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068084224/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907465:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907468:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907471:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765892.907478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765892.907499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061ddf80 x1876702068084352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068084352/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907527:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907530:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907532:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765892.907538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084480:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765892.907556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907597:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061dc380 x1876702068084480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907609:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068084480/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084480:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765892.907627:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765892.907648:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907659:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061de300 x1876702068084608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907669:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068084608/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907677:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907692:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068084608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765892.907701:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068084736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765892.907720:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907866:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061ddc00 x1876702068084736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907881:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068084736/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907889:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068084736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765892.907912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068084864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765892.907934:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765892.907946:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765892/real 1789765892] req@ffff9360061dd880 x1876702068084864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765892.907957:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068084864/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765892.907964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765892.907967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765892.907970:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068084864:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765893.290105:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765894.314826:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765895.338084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765896.369026:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765897.386323:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765898.026491:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026504:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765898.026510:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026524:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026539:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026547:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765898.026600:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068084992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026636:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026640:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765898.026642:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026646:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026650:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026654:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026666:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765898.026680:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068085120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026689:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026692:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765898.026694:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026698:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026713:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026718:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026723:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765898.026734:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068085248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026743:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026746:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765898.026748:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026752:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026755:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026759:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026764:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765898.026774:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068085376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026782:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026786:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765898.026787:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026791:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026794:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026834:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026839:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765898.026851:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068085504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026871:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026875:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765898.026877:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026881:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026884:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026888:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026893:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765898.026907:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068085632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765898.026917:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765898.026932:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765898.026934:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765898.026937:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3365 00000100:00080000:2.0:1789765898.026941:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765898.026944:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765898.026949:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765898.026959:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068085760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765898.028327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068084992:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765898.028423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061df100 x1876702068084992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028507:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068084992/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028519:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028527:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068084992:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765898.028536:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765898.028562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061df480 x1876702068085120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028589:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068085120/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028597:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765898.028610:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765898.028631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061dd500 x1876702068085248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028655:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068085248/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028668:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028671:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028674:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765898.028681:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085376:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765898.028716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028728:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061dc000 x1876702068085376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068085376/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085376:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765898.028759:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765898.028778:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028790:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061dce00 x1876702068085504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028868:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068085504/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028877:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028880:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028884:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068085504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765898.028891:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068085632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765898.028914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061dd180 x1876702068085632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.028943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068085632/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.028951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.028954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.028957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068085632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765898.028963:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068085760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765898.028984:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765898.028995:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765898/real 1789765898] req@ffff9360061df800 x1876702068085760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765898.029007:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068085760/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765898.029013:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765898.029016:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765898.029019:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068085760:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765898.410668:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765899.434017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765900.458589:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765901.484206:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765902.507148:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765903.146894:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.146903:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765903.146908:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.146916:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.146926:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.146933:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765903.146968:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068085888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147000:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147004:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765903.147005:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147009:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147012:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147015:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147023:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765903.147033:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068086016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147041:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147044:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765903.147045:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147048:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147050:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147054:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147057:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765903.147064:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068086144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147071:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147073:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765903.147074:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147077:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147079:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147082:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147085:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765903.147093:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068086272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147100:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147102:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765903.147104:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147106:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147108:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147111:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765903.147121:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068086400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147128:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147131:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765903.147132:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147134:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147137:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147141:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147145:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765903.147152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068086528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765903.147159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765903.147161:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765903.147162:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765903.147165:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3370 00000100:00080000:1.0:1789765903.147167:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765903.147170:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765903.147173:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765903.147179:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068086656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765903.147263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068085888:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765903.147320:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147366:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef4e00 x1876702068085888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147383:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068085888/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147392:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147395:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147399:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068085888:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765903.147406:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765903.147426:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147438:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef7100 x1876702068086016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147449:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068086016/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147457:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147463:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765903.147468:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765903.147486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147496:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef4700 x1876702068086144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068086144/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147528:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765903.147534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086272:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765903.147551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147561:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef5180 x1876702068086272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147571:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068086272/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147584:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086272:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765903.147589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086400:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765903.147603:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147611:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef5c00 x1876702068086400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068086400/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147633:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147636:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147638:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765903.147643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068086528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765903.147658:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef6d80 x1876702068086528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068086528/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068086528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765903.147696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068086656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765903.147713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765903.147724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765903/real 1789765903] req@ffff936010ef6300 x1876702068086656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765903.147735:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068086656/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765903.147741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765903.147744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765903.147747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068086656:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765903.530089:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765904.554173:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765905.578140:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765906.602177:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765907.628191:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765908.266083:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.266094:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765908.266098:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.266109:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.266119:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.266127:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765908.266165:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068086784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765908.266240:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068086784:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765908.266300:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765908.266345:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.266349:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765908.266351:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.266355:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.266359:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.266362:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.266371:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765908.266383:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068086912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765908.266635:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa5f80 x1876702068086784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.266652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068086784/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.266666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.266670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.266673:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068086784:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765908.266695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765908.266717:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.266815:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa7480 x1876702068086912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.266835:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068086912/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.266842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.266844:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.266932:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068086912:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765908.267682:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.267686:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765908.267688:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.267692:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.267696:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.267700:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.267707:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765908.267721:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068087040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765908.267740:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.267742:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765908.267744:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.267746:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.267748:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.267750:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.267752:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765908.267758:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068087168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765908.267763:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.267765:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765908.267766:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.267768:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.267769:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.267771:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.267773:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765908.267777:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068087296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765908.267782:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.267809:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765908.267810:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.267812:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.267815:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.267818:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.267893:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765908.267902:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068087424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765908.267908:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765908.267910:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765908.267911:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765908.267913:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3376 00000100:00080000:3.0:1789765908.267915:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765908.267917:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765908.267920:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765908.267924:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068087552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765908.267988:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765908.268014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.268025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa6680 x1876702068087040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.268034:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068087040/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.268040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.268042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.268044:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765908.268047:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087168:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765908.268059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.268065:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa5180 x1876702068087168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.268082:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068087168/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.268087:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.268088:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.268090:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087168:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765908.268094:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765908.268104:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.268111:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa4700 x1876702068087296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.268117:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068087296/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.268121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.268122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.268124:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765908.268127:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068087424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765908.268136:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.268141:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa4380 x1876702068087424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.268148:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068087424/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.268152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.268153:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.268155:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068087424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765908.268164:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068087552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765908.268176:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765908.268182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765908/real 1789765908] req@ffff936010aa4e00 x1876702068087552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765908.268188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068087552/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765908.268191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765908.268193:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765908.268195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068087552:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765908.651035:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765909.674011:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765910.698133:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765911.722783:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765912.746017:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765913.387116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387128:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765913.387133:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387143:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387154:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387162:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765913.387211:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068087680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387256:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387260:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765913.387262:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387266:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387269:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387273:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387282:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765913.387294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068087808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765913.387308:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387312:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387315:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387318:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387321:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765913.387331:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068087936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387343:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765913.387344:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387347:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387350:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387354:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387358:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765913.387366:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068088064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387375:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387377:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765913.387379:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387382:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387384:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387388:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387391:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765913.387401:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068088192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387408:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387411:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765913.387412:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387415:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387418:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387422:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387426:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765913.387434:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068088320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765913.387447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765913.387449:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765913.387451:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765913.387454:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3381 00000100:00080000:1.0:1789765913.387457:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765913.387460:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765913.387463:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765913.387471:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068088448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765913.387619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068087680:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765913.387678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.387723:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef4a80 x1876702068087680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.387741:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068087680/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.387760:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.387764:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.387768:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068087680:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765913.387775:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765913.387841:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.387855:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef5880 x1876702068087808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.387865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068087808/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.387873:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.387877:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.387879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765913.387887:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765913.387907:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.387919:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef4380 x1876702068087936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.387929:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068087936/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.387937:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.387939:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.387942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068087936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765913.387948:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088064:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765913.387967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.387978:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef4000 x1876702068088064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.387989:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068088064/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.387996:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.387999:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.388002:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088064:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765913.388008:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765913.388027:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.388038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef6300 x1876702068088192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.388049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068088192/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.388057:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.388060:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.388063:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765913.388069:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068088320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765913.388089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.388100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef6d80 x1876702068088320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.388111:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068088320/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.388119:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.388122:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.388125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068088320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765913.388130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068088448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765913.388149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765913.388160:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765913/real 1789765913] req@ffff936010ef5c00 x1876702068088448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765913.388171:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068088448/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765913.388185:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765913.388188:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765913.388191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068088448:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765913.774313:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765914.795775:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765915.818171:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765916.842209:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765917.866306:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765918.507652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.507662:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765918.507666:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.507676:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00080000:3.0:1789765918.507687:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.507694:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765918.507743:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068088576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765918.507778:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.507781:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765918.507783:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.507787:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00080000:3.0:1789765918.507791:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.507794:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.507838:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765918.507851:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068088704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765918.507860:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.507863:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765918.507865:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.507868:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00100000:0.0:1789765918.508068:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068088576:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765918.508126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765918.508180:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa7100 x1876702068088576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.508197:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068088576/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.508206:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.508209:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.508213:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068088576:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765918.508219:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765918.508238:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765918.508248:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa5500 x1876702068088704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.508258:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068088704/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.508264:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.508268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.508270:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088704:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765918.508626:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.508633:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.508638:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765918.508649:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068088832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765918.508664:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.508667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765918.508669:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.508677:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00080000:3.0:1789765918.508682:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.508685:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.508690:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765918.508697:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789765918.508700:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068088960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765918.508722:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.508725:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765918.508726:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.508730:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3386 00000800:00000100:0.0:1789765918.508731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765918.508733:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.508736:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.508741:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765918.508743:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa6a00 x1876702068088832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765918.508749:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068089088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765918.508755:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068088832/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765918.508758:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765918.508761:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765918.508762:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789765918.508763:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765918.508765:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789765918.508766:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00100000:0.0:1789765918.508769:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088832:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765918.508769:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765918.508772:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.508776:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765918.508779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765918.508784:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068089216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765918.508792:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789765918.508796:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765918.508796:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765918.508814:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765918.508818:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3386 00000100:00080000:3.0:1789765918.508822:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765918.508851:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa5500 x1876702068088960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.508863:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068088960/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.508871:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.508874:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.508877:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068088960:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765918.508883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765918.508902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765918.508914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa7100 x1876702068089088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.508925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068089088/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.508932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.508935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.508937:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765918.509014:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068089216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765918.509036:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765918.509052:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa4e00 x1876702068089216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.509062:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068089216/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.509069:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.509072:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.509075:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068089216:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765918.510368:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765918.510382:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765918.510409:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068089344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765918.512012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068089344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765918.512075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765918.512103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765918/real 1789765918] req@ffff936010aa4380 x1876702068089344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765918.512125:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068089344/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765918.512135:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765918.512139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765918.512143:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068089344:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765918.892057:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765919.920249:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765920.938010:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765921.964982:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765922.989496:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765923.625919:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.625928:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765923.625932:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765923.625942:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.625952:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.625959:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765923.626001:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068089472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765923.626087:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068089472:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765923.626144:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765923.626177:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.626183:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765923.626185:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765923.626189:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.626193:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.626198:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.626207:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765923.626219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068089600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765923.626386:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa4700 x1876702068089472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765923.626404:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068089472/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.626414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.626418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.626422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068089472:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765923.626446:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765923.626470:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765923.626483:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa5180 x1876702068089600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765923.626495:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068089600/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.626509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.626512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.626516:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089600:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765923.626915:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.626919:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765923.626921:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765923.626927:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.626932:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.626937:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.626943:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765923.626958:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068089728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765923.627006:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765923.627031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789765923.627667:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.627684:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765923.627687:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765923.627693:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa6680 x1876702068089728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765923.627693:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.627701:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.627709:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.627717:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765923.627720:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068089728/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.627732:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789765923.627737:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068089856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765923.627739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.627747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089728:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789765923.627759:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.627763:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765923.627765:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765923.627770:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089856:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789765923.627771:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.627778:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.627784:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.627790:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765923.627826:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765923.627830:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068089984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765923.627843:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765923.627844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa7480 x1876702068089856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765923.627849:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765923.627852:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765923.627859:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.627865:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765923.627866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068089856/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789765923.627871:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.627880:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765923.627882:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.627888:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.627893:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089856:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789765923.627898:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068090112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765923.627910:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765923.627913:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765923.627915:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765923.627919:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3391 00000100:00080000:3.0:1789765923.627923:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765923.627927:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765923.627931:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765923.627941:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068090240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765923.628035:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765923.628060:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765923.628073:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa5f80 x1876702068089984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765923.628084:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068089984/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.628092:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.628095:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.628098:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068089984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765923.628111:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068090112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765923.628132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765923.628144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa4a80 x1876702068090112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765923.628155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068090112/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.628163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.628166:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.628169:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068090112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765923.628175:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068090240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765923.628195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765923.628206:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765923/real 1789765923] req@ffff936010aa5880 x1876702068090240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765923.628217:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068090240/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765923.628230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765923.628234:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765923.628237:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068090240:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765924.009986:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765925.035704:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765926.058292:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765927.082357:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765928.107631:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765928.750249:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750259:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765928.750264:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750273:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750290:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750298:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765928.750334:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068090368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750374:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765928.750376:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750379:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750383:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750386:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765928.750408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068090496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750417:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750420:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765928.750422:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750425:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750427:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750431:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750434:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765928.750444:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068090624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750451:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765928.750455:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750458:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750460:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750463:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750466:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765928.750475:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068090752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750481:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750483:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765928.750485:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750488:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750491:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750494:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750497:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765928.750504:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068090880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750511:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750514:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765928.750516:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750519:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750521:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750524:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750528:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765928.750536:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068091008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765928.750544:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765928.750547:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765928.750549:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765928.750552:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3396 00000100:00080000:3.0:1789765928.750555:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765928.750557:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765928.750561:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765928.750569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068091136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765928.755019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068090368:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765928.755133:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765928.755159:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765928.755182:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765928.755202:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765928.755224:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765928.755256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765929.130152:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765930.158947:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765931.180842:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765931.818459:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765931.818513:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765931.818530:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818565:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818574:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818583:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818591:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818598:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765931.818605:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765931.818714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa6300 x1876702068090368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.818738:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068090368/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.818750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.818755:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.818759:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068090368:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765931.818766:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa7b80 x1876702068090496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.818778:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068090496/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.818987:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819001:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765931.819013:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa5c00 x1876702068090624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.819028:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068090624/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.819035:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819042:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765931.819048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa7800 x1876702068090752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.819059:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068090752/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.819067:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819071:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090752:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765931.819079:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa4000 x1876702068090880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.819091:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068090880/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.819272:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819280:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068090880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765931.819345:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa5880 x1876702068091008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.819363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068091008/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.819371:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819374:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819377:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765931.819384:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765928/real 1789765931] req@ffff936010aa4a80 x1876702068091136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765931.819395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068091136/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765931.819403:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765931.819406:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765931.819409:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091136:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765932.202067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765933.226774:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765933.868446:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765933.868456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765933.868460:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868470:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868479:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868487:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765933.868522:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068091264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765933.868551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765933.868555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765933.868557:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868562:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868565:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868569:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868578:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765933.868589:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068091392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765933.868609:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765933.868613:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765933.868614:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868618:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868622:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868626:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868630:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765933.868642:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068091520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765933.868653:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765933.868658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068091264:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765933.868658:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765933.868661:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868666:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868671:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868674:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868678:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765933.868690:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068091648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765933.868699:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765933.868702:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765933.868704:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868708:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868721:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868725:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868729:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765933.868733:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765933.868743:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068091776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765933.868753:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765933.868756:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765933.868758:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765933.868772:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868777:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868781:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868785:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765933.868826:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef5180 x1876702068091264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.868845:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068091264/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765933.868853:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068091904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765933.868858:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.868866:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789765933.868869:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789765933.868873:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068091264:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765933.868876:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765933.868880:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789765933.868885:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091392:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789765933.868889:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3401 00000100:00080000:1.0:1789765933.868894:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765933.868898:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765933.868904:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789765933.868911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765933.868917:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068092032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765933.869578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef4700 x1876702068091392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.869593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068091392/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.869613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.869616:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.869620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765933.869642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765933.869666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765933.869676:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef7100 x1876702068091520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.869687:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068091520/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.869695:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.869699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.869701:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765933.869707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091648:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765933.869723:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765933.869732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef4e00 x1876702068091648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.869742:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068091648/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.869750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.869753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.869756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091648:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765933.869761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765933.869776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765933.869785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef7b80 x1876702068091776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.869824:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068091776/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.869832:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.869834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.869837:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068091776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765933.869843:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765933.869860:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765933.869869:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef7480 x1876702068091904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.869879:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068091904/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.869886:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.869889:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.869892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068091904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765933.869897:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765933.869911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765933.870367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765933/real 1789765933] req@ffff936010ef6680 x1876702068092032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765933.870547:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068092032/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765933.870561:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765933.870565:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765933.870568:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092032:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765934.250050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765935.277360:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765936.299062:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765937.322327:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765938.346354:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765938.985930:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.985939:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765938.985943:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.985952:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.985961:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.985968:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765938.986000:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068092160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986034:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986038:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765938.986040:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986043:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986046:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986050:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986058:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765938.986068:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068092288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986083:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986086:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765938.986087:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986091:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986094:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986101:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765938.986109:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068092416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986117:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986120:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765938.986122:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986125:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986128:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986131:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986134:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765938.986141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068092544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765938.986154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986157:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986161:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986164:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986168:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765938.986176:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068092672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986184:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986187:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765938.986188:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986192:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986195:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986197:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986201:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765938.986217:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068092800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765938.986225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765938.986228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765938.986230:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765938.986233:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3406 00000100:00080000:1.0:1789765938.986236:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765938.986239:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765938.986243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765938.986250:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068092928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765938.986464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068092160:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765938.986517:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef7800 x1876702068092160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986585:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068092160/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986598:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986601:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068092160:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765938.986608:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765938.986627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986649:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef5500 x1876702068092288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986660:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068092288/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986667:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986670:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986674:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765938.986679:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765938.986697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986706:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef5f80 x1876702068092416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986717:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068092416/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986727:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765938.986736:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092544:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765938.986751:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986759:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef6a00 x1876702068092544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986781:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068092544/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986789:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986792:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986795:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092544:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765938.986827:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765938.986844:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986854:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef6680 x1876702068092672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986865:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068092672/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986873:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986876:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068092672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765938.986883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765938.986898:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986907:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef7480 x1876702068092800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986917:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068092800/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986924:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986927:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986930:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765938.986934:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765938.986948:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765938.986956:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765938/real 1789765938] req@ffff936010ef7b80 x1876702068092928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765938.986965:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068092928/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765938.986978:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765938.986981:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765938.986984:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068092928:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765939.373775:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765940.395104:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765941.423197:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765942.443891:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765943.466294:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765944.107614:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107624:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765944.107628:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107637:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107648:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107655:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765944.107689:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068093056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.107733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765944.107739:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107743:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107746:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107750:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765944.107760:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765944.107783:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068093184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.107835:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107838:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765944.107840:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107844:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107848:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107852:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765944.107856:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765944.107868:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068093312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.107877:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107880:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765944.107882:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107885:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107888:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107892:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765944.107896:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765944.107907:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068093440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.107916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107919:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765944.107920:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107923:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107927:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107930:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765944.107934:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765944.107955:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068093568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.107963:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.107967:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765944.107969:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.107981:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.107984:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.107988:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765944.107992:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765944.108001:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068093696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765944.108010:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765944.108012:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765944.108014:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765944.108017:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3411 00000100:00080000:2.0:1789765944.108019:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765944.108022:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108414:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068093056:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765944.108472:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108530:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061dfb80 x1876702068093056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108546:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068093056/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108555:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108562:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068093056:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765944.108568:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765944.108591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108604:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061de680 x1876702068093184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108615:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068093184/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108626:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765944.108635:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765944.108651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108660:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061dca80 x1876702068093312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108670:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068093312/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108677:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108679:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765944.108687:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093440:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765944.108702:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108710:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061df800 x1876702068093440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108719:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068093440/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093440:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765944.108747:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765944.108762:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108779:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061dd180 x1876702068093568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108789:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068093568/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108819:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108823:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108826:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068093568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765944.108832:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068093696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765944.108852:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.108863:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061dce00 x1876702068093696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.108873:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068093696/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.108881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.108883:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.108887:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068093696:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789765944.110337:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765944.110367:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068093824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765944.118837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068093824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765944.118914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765944.118946:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765944/real 1789765944] req@ffff9360061dc000 x1876702068093824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765944.118964:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068093824/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765944.118981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765944.118985:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765944.118989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068093824:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765944.490184:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765945.517248:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765946.549037:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765947.567358:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765948.587020:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765949.229628:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.229643:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765949.229648:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.229661:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.229676:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.229686:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765949.229892:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068093952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.229924:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.229928:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765949.229930:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.229934:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.229939:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.229944:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.229955:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765949.229969:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068094080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.229979:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.229982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765949.229985:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.229990:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.229993:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.229997:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.230002:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765949.230013:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068094208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.230022:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.230026:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765949.230028:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.230032:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.230035:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.230039:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.230044:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765949.230057:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068094336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.230065:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.230068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765949.230070:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.230074:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.230077:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.230081:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.230086:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765949.230095:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068094464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.230104:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.230107:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765949.230109:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.230113:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.230116:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.230120:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.230125:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765949.230134:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068094592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765949.230142:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765949.230145:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765949.230147:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765949.230150:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3417 00000100:00080000:0.0:1789765949.230153:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765949.230156:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765949.230161:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765949.230171:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068094720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765949.230648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068093952:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789765949.230731:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.230965:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020daf800 x1876702068093952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.230986:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068093952/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.230998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231003:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231008:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068093952:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765949.231015:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765949.231041:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020dadc00 x1876702068094080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068094080/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231075:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231078:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231082:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765949.231090:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765949.231109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231120:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020dadf80 x1876702068094208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231131:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068094208/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231139:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231142:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231144:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765949.231150:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094336:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765949.231170:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020daea00 x1876702068094336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068094336/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231202:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231205:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231209:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094336:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765949.231216:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765949.231234:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231245:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020dace00 x1876702068094464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231257:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068094464/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231265:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231268:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231271:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765949.231278:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068094592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765949.231297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231308:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020dae300 x1876702068094592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231321:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068094592/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231329:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231333:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231336:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068094592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765949.231341:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068094720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765949.231359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765949.231370:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765949/real 1789765949] req@ffff936020dad180 x1876702068094720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765949.231380:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068094720/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765949.231394:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765949.231397:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765949.231401:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068094720:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765949.610360:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765950.643844:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765951.658070:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765952.682856:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765953.707067:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765954.349896:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.349910:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765954.349915:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.349929:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.349943:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.349953:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765954.350004:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068094848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765954.350039:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350043:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765954.350045:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350050:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350054:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350058:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350070:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765954.350084:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068094976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765954.350095:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350099:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765954.350100:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350104:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350107:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350111:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350116:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765954.350126:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068095104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765954.350135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350137:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765954.350139:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350143:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350146:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350150:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350155:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765954.350169:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068095232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765954.350178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068094848:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789765954.350183:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350195:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765954.350197:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350203:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350206:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350209:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350214:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765954.350227:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068095360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765954.350246:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350250:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765954.350251:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350255:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350259:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350262:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350267:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789765954.350272:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789765954.350285:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068095488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765954.350296:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765954.350300:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765954.350301:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765954.350307:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3422 00000100:00080000:3.0:1789765954.350312:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765954.350317:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765954.350321:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765954.350330:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa5f80 x1876702068094848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789765954.350336:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068095616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765954.350375:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068094848/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.350389:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.350394:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.350398:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068094848:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765954.350409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765954.350436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.350461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa7480 x1876702068094976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.350472:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068094976/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.350479:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.350483:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.350487:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068094976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765954.350493:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765954.350513:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.350524:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa6680 x1876702068095104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.350535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068095104/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.350542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.350545:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.350548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765954.350576:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095232:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765954.350597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.350608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa5180 x1876702068095232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.350620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068095232/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.350629:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.350632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.350635:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095232:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765954.350643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765954.353566:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.353598:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa4700 x1876702068095360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.353616:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068095360/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.353628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.353632:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.353636:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765954.353646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068095488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765954.353668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.353680:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa4380 x1876702068095488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.353692:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068095488/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.353699:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.353702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.353706:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068095488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765954.353712:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068095616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765954.353730:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765954.353741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765954/real 1789765954] req@ffff936010aa4e00 x1876702068095616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765954.353750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068095616/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765954.353757:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765954.353760:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765954.353763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068095616:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765954.730191:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765955.759050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765956.783516:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765957.805992:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765958.829637:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765959.480491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765959.480505:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480514:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480525:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.480532:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765959.480569:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068095744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.480603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480607:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765959.480608:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480612:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480616:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.480620:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.480641:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765959.480654:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068095872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.480663:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480667:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765959.480668:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480672:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480675:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.480679:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.480683:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765959.480691:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068096000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.480700:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480703:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765959.480705:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480708:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480711:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.480714:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.480718:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765959.480725:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068096128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.480733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480736:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765959.480738:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480741:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480743:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.480747:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.480751:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765959.480758:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068096256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.480766:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.480769:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765959.480771:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.480774:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.480776:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765959.480886:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068095744:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765959.480946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.480990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef4e00 x1876702068095744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.481007:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068095744/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.481017:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.481021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.481024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068095744:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765959.481031:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765959.481050:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.481061:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef7100 x1876702068095872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.481071:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068095872/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.481079:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.481081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.481084:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068095872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765959.481090:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765959.481107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.481261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef4700 x1876702068096000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.481273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068096000/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.481286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.481289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.481292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765959.481298:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096128:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765959.481317:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.481327:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef5180 x1876702068096128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.481338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068096128/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.481345:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.481348:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.481352:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096128:192.168.203.137@tcp:8:kworker.0 00000100:00080000:1.0:1789765959.482882:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.482902:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765959.482935:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068096384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765959.482951:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765959.482955:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765959.482958:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765959.482962:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3427 00000100:00080000:1.0:1789765959.482970:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765959.482977:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765959.482982:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765959.482992:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068096512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765959.483086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765959.483147:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.483169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef5c00 x1876702068096256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.483185:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068096256/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.483194:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.483197:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.483201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765959.483230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068096384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765959.483251:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.483260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef6d80 x1876702068096384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.483270:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068096384/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.483277:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.483280:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.483283:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068096384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765959.483288:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068096512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765959.483303:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765959.483312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765959/real 1789765959] req@ffff936010ef6300 x1876702068096512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765959.483321:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068096512/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765959.483328:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765959.483331:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765959.483333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068096512:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789765959.850258:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765960.874001:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765961.900063:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765962.922045:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765963.946108:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765964.586636:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.586649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765964.586654:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.586666:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.586943:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.586952:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765964.586992:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068096640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587016:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765964.587018:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587021:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587025:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587028:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587037:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765964.587049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068096768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587057:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587061:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765964.587062:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587066:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587070:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587073:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587077:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765964.587087:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068096896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587094:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587097:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765964.587098:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587101:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587104:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587107:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587111:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765964.587119:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068097024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587126:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587129:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765964.587130:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587133:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587136:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587139:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587142:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765964.587152:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068097152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765964.587163:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587167:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587170:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587174:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587178:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765964.587187:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068097280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765964.587196:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765964.587199:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765964.587200:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765964.587204:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3432 00000100:00080000:1.0:1789765964.587207:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765964.587210:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765964.587214:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765964.587224:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068097408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765964.587326:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068096640:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765964.587403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.587612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef4000 x1876702068096640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.587631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068096640/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.587640:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.587644:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.587648:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068096640:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765964.587655:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765964.587959:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.587972:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef4380 x1876702068096768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.587985:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068096768/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.587992:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.587995:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.587999:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765964.588004:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765964.588021:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.588029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef5880 x1876702068096896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.588046:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068096896/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.588053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.588055:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.588058:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068096896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765964.588064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097024:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765964.588082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.588092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef4a80 x1876702068097024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.588103:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068097024/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.588111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.588114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.588117:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097024:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765964.588123:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765964.588140:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.588151:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef6300 x1876702068097152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.588161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068097152/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.588174:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.588178:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.588181:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765964.588186:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068097280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765964.588204:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.588215:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef6d80 x1876702068097280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.588226:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068097280/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.588235:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.588238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.588241:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068097280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765964.588247:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068097408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765964.588265:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765964.588275:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765964/real 1789765964] req@ffff936010ef5c00 x1876702068097408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765964.588285:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068097408/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765964.588292:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765964.588295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765964.588298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068097408:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765964.970007:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765966.000307:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765967.020223:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765968.042066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765969.070028:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765969.709921:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.709930:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765969.709935:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.709945:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.709955:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.709962:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765969.709997:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068097536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710032:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710036:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765969.710038:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710042:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710045:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710049:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710058:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765969.710070:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068097664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765969.710291:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710295:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710299:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710303:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710309:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765969.710322:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068097792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710341:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765969.710346:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710349:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710353:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710357:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710361:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765969.710371:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068097920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710381:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765969.710383:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710385:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710388:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710391:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765969.710403:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068098048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710411:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710414:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765969.710416:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710419:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710422:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710425:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710429:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765969.710439:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068098176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765969.710452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765969.710454:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765969.710456:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765969.710463:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3437 00000100:00080000:1.0:1789765969.710467:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765969.710470:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765969.710474:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765969.710483:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068098304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765969.710791:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068097536:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765969.710875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.710911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef5180 x1876702068097536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.710926:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068097536/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.710935:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.710939:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.710942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068097536:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765969.710949:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765969.710970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.710995:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef4700 x1876702068097664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.711007:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068097664/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.711015:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.711018:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.711021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765969.711029:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765969.711057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.711069:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef7100 x1876702068097792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.711081:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068097792/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.711088:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.711091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.711094:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765969.711100:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097920:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765969.711118:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.711141:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef4e00 x1876702068097920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.711152:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068097920/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.711160:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.711163:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.711165:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068097920:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765969.711171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098048:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765969.711189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.711200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef7b80 x1876702068098048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.711210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068098048/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.711218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.711221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.711223:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765969.711228:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068098176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765969.712683:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.712714:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef7480 x1876702068098176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.712732:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068098176/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.712741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.712745:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.712749:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068098176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765969.712756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068098304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765969.712787:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765969.712811:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765969/real 1789765969] req@ffff936010ef6680 x1876702068098304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765969.712823:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068098304/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765969.712836:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765969.712838:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765969.712841:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068098304:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765970.090046:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765971.114362:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765972.138104:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765973.162653:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765974.187668:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789765974.826284:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826303:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789765974.826308:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826317:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826327:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826334:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789765974.826373:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068098432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826408:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826412:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765974.826413:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826417:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826421:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826425:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826434:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765974.826446:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068098560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826455:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826459:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765974.826460:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826464:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826468:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826471:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826475:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765974.826486:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068098688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826495:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826498:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789765974.826500:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826503:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826506:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826510:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826514:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789765974.826525:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068098816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826533:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826537:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789765974.826538:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826541:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826545:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826548:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826552:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765974.826561:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068098944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826570:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826573:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789765974.826574:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826578:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826581:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826584:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826589:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765974.826598:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068099072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789765974.826606:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789765974.826610:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789765974.826611:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789765974.826614:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3442 00000100:00080000:3.0:1789765974.826617:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789765974.826621:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789765974.826625:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789765974.826634:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068099200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765974.826733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068098432:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765974.826795:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.826891:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa7100 x1876702068098432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.826909:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068098432/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.826920:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.826923:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.826927:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068098432:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765974.826934:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765974.826957:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.826970:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa5500 x1876702068098560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.826981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068098560/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.826995:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.826998:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827001:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765974.827006:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765974.827025:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.827035:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa6a00 x1876702068098688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.827046:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068098688/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.827054:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.827057:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765974.827066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098816:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765974.827086:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.827097:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa4e00 x1876702068098816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.827107:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068098816/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.827114:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.827116:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827120:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098816:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765974.827125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765974.827142:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.827152:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa4380 x1876702068098944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.827161:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068098944/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.827168:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.827170:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827173:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068098944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765974.827178:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765974.827195:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.827205:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa4700 x1876702068099072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.827214:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068099072/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.827221:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.827224:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827226:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765974.827231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765974.827250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765974.827262:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765974/real 1789765974] req@ffff936010aa5180 x1876702068099200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765974.827273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068099200/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789765999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765974.827281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765974.827284:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765974.827287:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099200:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765975.210080:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765976.234190:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765977.260451:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765978.282172:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765979.306060:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789765979.949133:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949144:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789765979.949149:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949159:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00080000:2.0:1789765979.949169:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949177:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789765979.949218:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068099328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765979.949251:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765979.949264:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949269:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00080000:2.0:1789765979.949273:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949285:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765979.949297:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068099456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765979.949306:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949309:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765979.949311:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949314:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00080000:2.0:1789765979.949319:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949322:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949326:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765979.949336:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068099584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765979.949345:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949347:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789765979.949349:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949353:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00100000:1.0:1789765979.949355:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068099328:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789765979.949356:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949359:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949364:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789765979.949371:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068099712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765979.949379:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789765979.949383:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949386:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00080000:2.0:1789765979.949388:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949391:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949394:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765979.949403:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068099840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789765979.949409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789765979.949410:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949413:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789765979.949414:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949417:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00080000:2.0:1789765979.949420:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949423:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949427:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789765979.949435:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068099968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789765979.949447:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789765979.949450:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789765979.949451:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789765979.949455:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3447 00000100:00100000:1.0:1789765979.949456:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061dd500 x1876702068099328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789765979.949457:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789765979.949461:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789765979.949464:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765979.949472:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068099328/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789765979.949472:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068100096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765979.949482:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949486:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068099328:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789765979.949496:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765979.949518:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061df480 x1876702068099456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949542:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068099456/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949549:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949554:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765979.949559:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765979.949576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949586:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061df100 x1876702068099584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949595:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068099584/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949602:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949605:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949608:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765979.949631:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099712:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765979.949650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949661:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061dd880 x1876702068099712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949676:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068099712/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099712:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765979.949694:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789765979.949712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949722:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061ddc00 x1876702068099840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949731:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068099840/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949738:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949740:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068099840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765979.949747:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765979.949763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061de300 x1876702068099968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949781:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068099968/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949794:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949797:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949800:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068099968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765979.949805:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789765979.949857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789765979.949868:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765979/real 1789765979] req@ffff9360061dc380 x1876702068100096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789765979.949878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068100096/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765979.949885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789765979.949888:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765979.949891:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100096:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765980.330249:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765981.355868:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765982.378052:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765983.402249:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765984.426103:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765985.065975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.065985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765985.065989:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.065999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066008:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066015:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765985.066049:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068100224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765985.066077:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765985.066083:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066087:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066091:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066095:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066104:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765985.066116:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068100352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765985.066125:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068100224:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789765985.066129:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066135:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765985.066137:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066141:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066149:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066154:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765985.066166:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068100480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765985.066176:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765985.066182:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066185:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066188:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066191:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066196:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765985.066206:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068100608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765985.066215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765985.066228:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066233:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066236:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066240:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066244:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765985.066254:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068100736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789765985.066264:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789765985.066266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066270:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765985.066272:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066277:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066281:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765985.066285:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066291:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765985.066303:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068100864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765985.066312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef6a00 x1876702068100224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765985.066314:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765985.066322:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765985.066327:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765985.066334:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3452 00000100:00080000:1.0:1789765985.066338:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789765985.066343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068100224/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789765985.066345:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765985.066352:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789765985.066357:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066362:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789765985.066367:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068100992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765985.066369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068100224:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765985.066399:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765985.066493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.066510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef5f80 x1876702068100352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.066521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068100352/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.066529:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066532:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.066535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765985.066541:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765985.066561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.066571:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef5500 x1876702068100480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.066582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068100480/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.066590:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066593:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.066596:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765985.066603:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100608:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765985.066622:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.066633:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef7800 x1876702068100608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.066659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068100608/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.066666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.066672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100608:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765985.066677:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765985.066697:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.066708:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef6680 x1876702068100736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.066718:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068100736/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.066725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066728:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.066731:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068100736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765985.066918:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100864:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765985.066942:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.066953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef7480 x1876702068100864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.066965:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068100864/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.066979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.066983:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.066985:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765985.066991:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765985.067010:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765985.067021:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765985/real 1789765985] req@ffff936010ef7b80 x1876702068100992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765985.067030:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068100992/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765985.067038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765985.067041:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765985.067043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068100992:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789765985.451355:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765986.474063:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765987.500188:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765988.522052:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765989.546093:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789765990.186275:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186285:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789765990.186289:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186299:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186310:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186317:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789765990.186353:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068101120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186385:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186388:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765990.186390:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186393:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186396:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186400:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186409:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765990.186420:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068101248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186430:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186433:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765990.186434:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186438:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186440:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186443:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186447:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765990.186456:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068101376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186463:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789765990.186467:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186471:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186474:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186477:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186481:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789765990.186489:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068101504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186499:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789765990.186501:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186504:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186507:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186510:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186515:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765990.186523:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068101632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186532:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186535:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789765990.186536:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186540:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186542:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186546:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186550:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765990.186561:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068101760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789765990.186569:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789765990.186571:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789765990.186572:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789765990.186576:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3458 00000100:00080000:1.0:1789765990.186578:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789765990.186581:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789765990.186585:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765990.186593:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068101888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789765990.186685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068101120:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789765990.186743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.186783:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef4e00 x1876702068101120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.186832:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068101120/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.186848:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.186852:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.186856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068101120:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765990.186864:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765990.186887:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.186901:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef7100 x1876702068101248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.186912:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068101248/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.186919:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.186922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.186924:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765990.186930:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765990.186946:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.186971:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef4700 x1876702068101376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.186981:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068101376/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.186988:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.186991:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.186994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765990.187000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101504:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765990.187019:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.187029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef5180 x1876702068101504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.187040:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068101504/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.187047:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.187050:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.187053:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101504:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765990.187057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789765990.187073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.187081:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef5c00 x1876702068101632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.187092:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068101632/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.187098:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.187101:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.187104:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068101632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765990.187109:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068101760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765990.187125:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.187135:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef6d80 x1876702068101760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.187146:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068101760/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.187153:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.187155:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.187157:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068101760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765990.187161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068101888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789765990.187179:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789765990.187207:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765990/real 1789765990] req@ffff936010ef6300 x1876702068101888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765990.187217:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068101888/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765990.187225:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765990.187227:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765990.187230:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068101888:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765990.570087:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765991.596633:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789765992.618139:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765993.642052:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765994.666658:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789765995.307372:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307382:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789765995.307395:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307405:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307416:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307423:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789765995.307468:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068102016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307502:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307506:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765995.307508:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307512:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307515:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307518:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307527:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765995.307539:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068102144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307549:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307552:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765995.307554:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307558:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307561:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307564:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307568:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765995.307577:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068102272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307586:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307588:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789765995.307590:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307593:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307596:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307599:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307603:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789765995.307611:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068102400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307619:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307621:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789765995.307623:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307626:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307628:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307631:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307634:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765995.307642:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068102528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307651:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307654:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789765995.307656:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307660:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307663:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307667:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307672:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789765995.307679:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068102016:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789765995.307683:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068102656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789765995.307694:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789765995.307697:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789765995.307698:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789765995.307702:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3463 00000100:00080000:0.0:1789765995.307706:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789765995.307709:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789765995.307713:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789765995.307723:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068102784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789765995.307779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765995.307844:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765995.307884:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765995.307907:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789765995.307950:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068102656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789765995.307977:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068102784:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765995.698120:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765996.719110:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789765997.738641:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789765998.378109:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789765998.379365:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789765998.379384:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765998.379621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020dac380 x1876702068102016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766020 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.379646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068102016/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766020 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.379658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.379663:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.379667:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068102016:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789765998.379722:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765998.379793:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020dafb80 x1876702068102144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.379849:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068102144/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.379864:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.379867:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000800:00000100:2.0:1789765998.379867:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765998.379871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765998.379878:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020daf480 x1876702068102272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789765998.379878:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789765998.379900:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789765998.379941:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068102272/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.379950:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.379954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.379956:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765998.379963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020daed80 x1876702068102400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.379975:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068102400/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789765998.379977:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:0.0:1789765998.379984:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000800:00000100:2.0:1789765998.379986:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00080000:0.0:1789765998.379987:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.379990:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102400:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765998.379996:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020dae680 x1876702068102528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.380006:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068102528/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.380014:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.380017:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.380020:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068102528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789765998.380036:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020dac000 x1876702068102656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.380047:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068102656/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.380055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.380058:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.380061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068102656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789765998.380067:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789765995/real 1789765998] req@ffff936020dad880 x1876702068102784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789765998.380076:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068102784/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789765998.380082:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789765998.380085:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789765998.380087:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068102784:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789765998.762012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789765999.788645:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766000.426115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766000.426124:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766000.426128:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426138:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426148:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426155:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766000.426188:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068102912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766000.426215:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766000.426219:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766000.426221:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426225:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426228:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426232:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426241:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766000.426253:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068103040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766000.426263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789766000.426267:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068102912:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766000.426268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766000.426272:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426279:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426283:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426286:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426290:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766000.426304:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068103168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766000.426313:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766000.426316:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766000.426318:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426322:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426326:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426331:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789766000.426332:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426338:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766000.426350:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068103296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766000.426359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766000.426362:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766000.426364:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426368:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426372:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766000.426375:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061ddf80 x1876702068102912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766000.426388:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426395:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766000.426401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068102912/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766000.426407:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068103424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766000.426415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766000.426419:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766000.426423:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:2.0:1789766000.426424:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766000.426428:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789766000.426430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068102912:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766000.426435:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426440:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426444:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426448:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766000.426457:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068103552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766000.426460:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103040:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766000.426474:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766000.426477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766000.426478:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766000.426483:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3468 00000100:00080000:2.0:1789766000.426486:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766000.426490:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789766000.426495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:2.0:1789766000.426496:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766000.426507:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068103680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766000.426517:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061dc700 x1876702068103040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068103040/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426543:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426546:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766000.426553:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766000.426571:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766000.426594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061dc380 x1876702068103168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068103168/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426625:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426627:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766000.426632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103296:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766000.426651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766000.426661:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061de300 x1876702068103296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426671:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068103296/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426678:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103296:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766000.426688:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766000.426704:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766000.426715:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061ddc00 x1876702068103424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426725:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068103424/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426730:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426733:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426736:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766000.426751:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068103552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766000.426777:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766000.426788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061dd880 x1876702068103552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426819:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068103552/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426826:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426829:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426831:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068103552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766000.426836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068103680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766000.426855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766000.426865:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766000/real 1789766000] req@ffff9360061df100 x1876702068103680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766000.426876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068103680/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766000.426883:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766000.426886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766000.426888:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068103680:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766000.810164:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766001.834948:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766002.858144:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766003.882976:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766004.906795:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766005.546283:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546293:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766005.546297:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546306:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546317:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546324:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766005.546361:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068103808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546391:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766005.546398:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546401:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546414:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546418:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546427:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766005.546439:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068103936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546451:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766005.546453:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546456:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546460:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546463:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546467:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766005.546476:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068104064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546488:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546491:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766005.546493:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546496:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546499:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546503:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546507:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766005.546517:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068104192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546525:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546528:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766005.546530:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546533:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546536:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546539:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546543:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766005.546554:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068104320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546565:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766005.546566:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546569:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546572:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546575:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546579:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766005.546589:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068104448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766005.546597:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766005.546600:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766005.546601:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766005.546604:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3473 00000100:00080000:1.0:1789766005.546608:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766005.546610:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766005.546615:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766005.546624:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068104576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766005.546953:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068103808:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766005.547012:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547060:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef4a80 x1876702068103808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547085:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068103808/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547095:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547098:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547102:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068103808:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766005.547110:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766005.547132:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547145:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef5880 x1876702068103936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547156:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068103936/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547163:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547166:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547169:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068103936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766005.547175:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766005.547194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547204:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef4380 x1876702068104064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547220:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068104064/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547227:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547229:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547232:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766005.547236:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104192:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766005.547256:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef4000 x1876702068104192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547278:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068104192/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547285:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547288:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547291:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104192:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766005.547297:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766005.547316:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547326:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef6300 x1876702068104320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547337:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068104320/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547349:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547352:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547355:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766005.547361:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068104448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766005.547379:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547390:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef6d80 x1876702068104448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068104448/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547408:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068104448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766005.547419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068104576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766005.547439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766005.547449:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766005/real 1789766005] req@ffff936010ef5c00 x1876702068104576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766005.547460:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068104576/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766005.547467:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766005.547470:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766005.547473:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068104576:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766005.931556:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766006.954091:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766007.978303:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766009.002976:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766010.030125:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766010.667052:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667062:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766010.667066:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667075:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667086:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667093:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766010.667128:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068104704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667163:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667166:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766010.667168:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667171:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667175:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667178:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667187:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766010.667198:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068104832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667207:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766010.667212:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667216:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667220:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667223:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667227:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766010.667237:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068104960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667245:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667249:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766010.667251:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667254:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667257:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667261:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667265:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766010.667273:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068105088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667281:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667283:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766010.667284:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667287:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667291:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667294:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667297:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766010.667306:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068105216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667313:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667316:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766010.667318:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667321:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667323:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667326:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667330:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766010.667339:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068105344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766010.667347:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766010.667350:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766010.667351:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766010.667354:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3478 00000100:00080000:3.0:1789766010.667358:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766010.667361:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766010.667365:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766010.667383:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068105472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766010.667485:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068104704:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766010.667547:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa6680 x1876702068104704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068104704/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667623:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068104704:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766010.667630:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766010.667651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667663:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa7480 x1876702068104832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667673:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068104832/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667680:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766010.667689:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766010.667721:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa5f80 x1876702068104960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667745:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068104960/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667752:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667755:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667757:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068104960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766010.667762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105088:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766010.667781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667791:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa4a80 x1876702068105088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667834:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068105088/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667842:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667845:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667848:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105088:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766010.667854:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766010.667875:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa5880 x1876702068105216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667903:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068105216/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667911:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667914:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766010.667922:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068105344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766010.667941:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.667953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa4000 x1876702068105344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.667963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068105344/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.667970:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.667973:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.667976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068105344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766010.667981:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068105472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766010.668001:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766010.668011:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766010/real 1789766010] req@ffff936010aa7800 x1876702068105472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766010.668021:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068105472/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766010.668044:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766010.668047:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766010.668050:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068105472:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766011.050297:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766012.078707:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766013.100558:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766014.125144:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766015.147972:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766015.786562:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786576:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766015.786581:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786595:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786618:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786627:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766015.786681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068105600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786715:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786719:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766015.786721:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786726:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786730:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786734:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786747:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766015.786760:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068105728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786769:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786772:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766015.786774:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786778:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786781:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786784:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786788:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766015.786841:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068105856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786851:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786854:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766015.786855:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786859:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786862:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786866:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786871:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766015.786881:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068105984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786888:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786891:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766015.786893:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786896:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786899:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786902:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786905:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766015.786916:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068106112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786923:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766015.786928:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786931:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786934:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786938:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786942:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766015.786952:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068106240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766015.786959:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766015.786962:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766015.786964:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766015.786968:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3483 00000100:00080000:1.0:1789766015.786971:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766015.786975:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766015.786979:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766015.786988:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068106368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766015.787209:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068105600:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766015.787302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787354:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef5180 x1876702068105600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787374:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068105600/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787385:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787389:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787394:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068105600:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766015.787401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766015.787425:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787438:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef4700 x1876702068105728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787450:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068105728/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787458:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787461:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787464:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766015.787470:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766015.787491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787502:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef7100 x1876702068105856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068105856/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787522:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787528:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766015.787534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105984:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766015.787555:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef4e00 x1876702068105984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068105984/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787613:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068105984:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766015.787619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766015.787642:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef7b80 x1876702068106112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787666:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068106112/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787674:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787677:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787680:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766015.787685:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068106240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766015.787705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787717:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef7480 x1876702068106240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068106240/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068106240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766015.787746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068106368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766015.787766:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766015.787777:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766015/real 1789766015] req@ffff936010ef6680 x1876702068106368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766015.787795:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068106368/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766015.787866:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766015.787870:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766015.787873:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068106368:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766016.170832:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766017.195934:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766018.227557:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766019.242592:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766020.268641:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766020.906004:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906013:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766020.906018:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906027:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906038:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906044:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766020.906083:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068106496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906114:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906117:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766020.906119:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906123:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906126:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906130:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906138:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766020.906150:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068106624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906160:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906163:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766020.906165:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906169:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906172:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906175:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906179:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766020.906189:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068106752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906200:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906203:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766020.906205:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906208:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906211:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906214:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906218:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766020.906228:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068106880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906236:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906239:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766020.906240:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906243:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906247:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906250:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906254:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766020.906264:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068107008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906272:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906275:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766020.906276:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906280:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906283:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906286:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906290:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766020.906300:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068107136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766020.906308:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766020.906311:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766020.906312:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766020.906316:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3488 00000100:00080000:3.0:1789766020.906319:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766020.906324:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766020.906328:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766020.906335:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068107264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766020.907011:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068106496:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766020.907080:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.907124:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa5c00 x1876702068106496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.907141:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068106496/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.907152:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.907156:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.907159:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068106496:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766020.907166:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766020.907189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.907200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa7b80 x1876702068106624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.907211:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068106624/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.907218:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.907221:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.907224:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766020.907230:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766020.907248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.907259:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa6300 x1876702068106752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.907269:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068106752/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.907276:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.907279:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.907283:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766020.907289:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106880:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766020.907308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.907319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa7800 x1876702068106880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.908023:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068106880/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.908036:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.908039:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.908043:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068106880:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766020.908051:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766020.908083:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.908098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa4000 x1876702068107008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.908111:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068107008/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.908119:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.908121:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.908124:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766020.908131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068107136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766020.908152:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.908162:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa5880 x1876702068107136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.908173:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068107136/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.908179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.908182:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.908185:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068107136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766020.908190:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068107264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766020.908208:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766020.908219:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766020/real 1789766020] req@ffff936010aa4a80 x1876702068107264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766020.908228:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068107264/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766020.908236:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766020.908238:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766020.908240:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068107264:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766021.291895:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766022.314777:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766023.339075:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766024.364414:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766025.386141:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766026.026474:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026484:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766026.026488:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026498:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026509:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026516:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766026.026552:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068107392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026588:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026592:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766026.026594:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026597:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026601:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026604:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026613:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766026.026626:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068107520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026636:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026639:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766026.026641:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026644:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026648:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026652:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026656:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766026.026667:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068107648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026676:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026679:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766026.026681:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026684:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026687:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026690:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026695:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766026.026704:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068107776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026713:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026716:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766026.026718:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026721:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026724:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026727:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026731:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766026.026741:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068107904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026749:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026752:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766026.026766:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026770:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026773:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026776:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026780:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766026.026789:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068108032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766026.026832:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766026.026835:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766026.026837:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766026.026841:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3493 00000100:00080000:0.0:1789766026.026844:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766026.026847:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766026.026852:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766026.027019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068107392:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766026.027075:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020dad880 x1876702068107392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068107392/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027138:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027142:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027145:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068107392:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766026.027152:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766026.027174:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027185:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020dac000 x1876702068107520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027195:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068107520/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027206:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027208:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766026.027214:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766026.027232:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027243:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020dae680 x1876702068107648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068107648/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027260:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027262:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027266:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766026.027271:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107776:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766026.027289:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027297:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020daed80 x1876702068107776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027308:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068107776/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027314:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027319:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107776:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766026.027324:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766026.027339:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020daf480 x1876702068107904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068107904/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027365:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027367:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027370:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068107904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766026.027374:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766026.027389:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027397:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020dafb80 x1876702068108032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027407:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068108032/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027413:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027415:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027418:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766026.027483:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068108160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766026.027696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766026.027718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766026.027729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766026/real 1789766026] req@ffff936020dad500 x1876702068108160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766026.027739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068108160/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766026.027759:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766026.027762:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766026.027765:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108160:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766026.412617:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766027.434425:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766028.458312:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766029.482088:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766030.509967:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766031.146117:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146127:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766031.146131:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146141:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146151:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146159:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766031.146196:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068108288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146228:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146232:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766031.146234:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146238:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146241:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146245:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146254:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766031.146266:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068108416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146275:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146278:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766031.146281:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146284:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146287:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146291:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146295:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766031.146305:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068108544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146315:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146318:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766031.146320:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146323:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146326:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146329:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146334:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766031.146343:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068108672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146351:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146354:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766031.146356:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146359:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146362:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146365:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146369:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766031.146378:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068108800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146387:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146389:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766031.146391:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146394:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146398:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146402:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146406:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766031.146417:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068108928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766031.146425:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766031.146427:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766031.146429:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766031.146432:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3498 00000100:00080000:3.0:1789766031.146435:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766031.146438:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766031.146442:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766031.146451:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068109056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766031.147982:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068108288:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766031.148057:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148098:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa5f80 x1876702068108288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068108288/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148125:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148129:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148132:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068108288:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766031.148139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766031.148161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148173:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa7480 x1876702068108416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068108416/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148197:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766031.148204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766031.148222:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148233:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa6680 x1876702068108544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148244:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068108544/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148251:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148254:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148257:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766031.148263:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108672:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766031.148282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148292:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa5180 x1876702068108672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148303:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068108672/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148310:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148313:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148316:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108672:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766031.148322:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766031.148340:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148351:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa4700 x1876702068108800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148361:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068108800/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148369:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148372:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148375:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068108800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766031.148380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766031.148397:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148407:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa4380 x1876702068108928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068108928/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148424:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148427:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148430:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068108928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766031.148436:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766031.148453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766031.148463:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766031/real 1789766031] req@ffff936010aa4e00 x1876702068109056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766031.148475:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068109056/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766031.148483:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766031.148486:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766031.148489:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109056:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766031.530009:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766032.554009:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766033.578616:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766034.602123:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766035.626202:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766036.269563:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269573:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766036.269577:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269587:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269597:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269604:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766036.269641:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068109184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269678:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269682:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766036.269684:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269688:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269691:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269695:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269704:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766036.269717:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068109312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269725:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269728:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766036.269731:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269734:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269737:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269740:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269744:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766036.269754:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068109440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269762:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269765:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766036.269767:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269770:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269773:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269777:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269781:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766036.269790:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068109568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269858:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269861:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766036.269863:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269866:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269870:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269873:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269877:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766036.269888:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068109696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269897:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269899:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766036.269901:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269904:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269907:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269911:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269915:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766036.269925:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068109824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766036.269932:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766036.269935:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766036.269936:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766036.269939:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3504 00000100:00080000:0.0:1789766036.269942:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766036.269946:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766036.269950:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766036.269958:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068109952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766036.273112:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068109184:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766036.273187:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273244:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020daf100 x1876702068109184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273262:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068109184/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273273:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273281:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068109184:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766036.273289:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766036.273311:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273324:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020dac700 x1876702068109312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273336:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068109312/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273343:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273346:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273350:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766036.273356:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766036.273376:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020dad500 x1876702068109440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068109440/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273407:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273409:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766036.273415:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109568:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766036.273435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273445:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020dafb80 x1876702068109568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068109568/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273465:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273471:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109568:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766036.273476:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109696:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766036.273496:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020daf480 x1876702068109696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068109696/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273524:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273527:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273530:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068109696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766036.273535:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766036.273553:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273565:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020daed80 x1876702068109824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068109824/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273589:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766036.273594:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109952:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766036.273614:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766036.273626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766036/real 1789766036] req@ffff936020dae680 x1876702068109952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766036.273643:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068109952/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766036.273650:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766036.273652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766036.273656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068109952:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766036.650662:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766037.676604:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766038.698130:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766039.722023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766040.746699:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766041.385965:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.385975:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766041.385979:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.385989:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386000:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386007:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766041.386047:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068110080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766041.386081:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386084:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766041.386086:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.386090:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386094:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766041.386105:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766041.386119:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068110208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766041.386129:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068110080:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789766041.386143:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386146:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766041.386148:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.386154:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386158:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386162:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766041.386167:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766041.386180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068110336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789766041.386194:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789766041.386201:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386205:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766041.386207:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.386212:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386217:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386221:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766041.386226:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766041.386242:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dac000 x1876702068110080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766041.386243:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068110464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766041.386271:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386275:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766041.386277:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766041.386280:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068110080/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766041.386284:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386291:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386296:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789766041.386297:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789766041.386303:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789766041.386304:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386310:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068110080:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766041.386319:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068110592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766041.386329:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386333:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789766041.386335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110208:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789766041.386336:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.386345:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386350:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386355:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766041.386360:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789766041.386366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766041.386375:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068110720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766041.386383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dad880 x1876702068110208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766041.386389:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766041.386393:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766041.386395:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766041.386400:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3509 00000100:00080000:0.0:1789766041.386403:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766041.386409:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766041.386414:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766041.386415:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068110208/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386426:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789766041.386428:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068110848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766041.386432:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386439:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766041.386452:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766041.386473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766041.386484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dac380 x1876702068110336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766041.386494:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068110336/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386501:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766041.386504:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386507:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766041.386511:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110464:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766041.386529:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766041.386539:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dad180 x1876702068110464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766041.386549:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068110464/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386556:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766041.386558:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386562:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110464:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766041.386566:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766041.386584:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766041.386595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dae300 x1876702068110592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766041.386606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068110592/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766041.386626:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068110592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766041.386643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068110720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766041.386660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766041.386670:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020dace00 x1876702068110720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766041.386680:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068110720/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386687:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766041.386690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068110720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766041.386698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068110848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766041.386722:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766041.386733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766041/real 1789766041] req@ffff936020daea00 x1876702068110848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766041.386743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068110848/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766041.386750:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766041.386753:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766041.386756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068110848:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766041.771386:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766042.794005:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766043.819967:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766044.842077:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766045.866989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766046.509214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.509224:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766046.509229:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.509238:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.509249:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.509257:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766046.509292:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068110976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.509322:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.509326:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766046.509327:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.509331:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.509334:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.509338:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.509347:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766046.509358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068111104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.509369:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.509372:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766046.509373:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.509377:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.509380:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.509383:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.509387:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766046.509397:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068111232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.509405:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.509408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766046.509410:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.509413:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00100000:0.0:1789766046.509512:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068110976:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766046.509576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.509620:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef7800 x1876702068110976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.509637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068110976/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.509646:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.509650:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.509654:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068110976:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766046.509661:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766046.509686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.509698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef5500 x1876702068111104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.509709:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068111104/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.509717:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.509721:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.509724:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766046.509730:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766046.509749:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.509760:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef5f80 x1876702068111232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.509784:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068111232/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.509792:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.509795:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.509837:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111232:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766046.510223:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.510232:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.510238:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766046.510255:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068111360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.510277:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.510280:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766046.510282:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.510287:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.510292:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.510296:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.510300:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766046.510310:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068111488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.510318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.510321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766046.510323:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.510326:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.510329:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.510332:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.510336:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766046.510345:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068111616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766046.510353:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766046.510356:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766046.510358:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766046.510361:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3514 00000100:00080000:1.0:1789766046.510364:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766046.510367:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766046.510371:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766046.510379:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068111744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766046.510460:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111360:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766046.510488:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.510503:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef6a00 x1876702068111360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.510516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068111360/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.510524:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.510528:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.510531:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111360:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766046.510537:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766046.510556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.510567:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef5f80 x1876702068111488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.510579:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068111488/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.510586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.510588:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.510591:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068111488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766046.510596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068111616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766046.510614:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.510624:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef5500 x1876702068111616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.510642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068111616/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.510649:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.510652:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.510655:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068111616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766046.510660:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068111744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766046.510679:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766046.510689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766046/real 1789766046] req@ffff936010ef7800 x1876702068111744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766046.510698:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068111744/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766046.510704:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766046.510706:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766046.510709:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068111744:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766046.890639:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766047.914198:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766048.938551:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766049.962091:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766050.986996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766051.625874:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.625884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766051.625888:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.625897:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.625908:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.625914:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766051.625948:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068111872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766051.625979:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.625983:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766051.625984:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.625988:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.625991:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.625994:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766051.626002:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766051.626013:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068112000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766051.626022:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.626026:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766051.626028:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.626032:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.626036:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.626041:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626043:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068111872:192.168.203.137@tcp:250:kworker.0 00000008:00100000:1.0:1789766051.626046:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766051.626060:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068112128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766051.626069:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.626072:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766051.626074:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.626078:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.626082:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.626085:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766051.626089:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766051.626099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068112256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766051.626108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789766051.626112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766051.626113:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766051.626117:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.626121:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.626126:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.626129:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766051.626135:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766051.626146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068112384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766051.626155:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef6680 x1876702068111872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766051.626172:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.626176:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766051.626178:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.626183:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00100000:0.0:1789766051.626184:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068111872/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766051.626189:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.626196:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789766051.626203:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789766051.626203:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789766051.626212:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626217:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068111872:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766051.626220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068112512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766051.626227:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112000:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766051.626557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766051.626560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766051.626562:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766051.626566:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3519 00000100:00080000:1.0:1789766051.626571:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766051.626575:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766051.626580:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766051.626592:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068112640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766051.626652:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626666:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef7480 x1876702068112000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.626678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068112000/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.626686:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.626689:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766051.626713:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766051.626741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef7b80 x1876702068112128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.626763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068112128/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.626771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.626774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112128:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766051.626783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112256:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766051.626831:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626843:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef4e00 x1876702068112256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.626855:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068112256/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.626863:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.626866:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626869:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112256:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766051.626875:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766051.626895:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef7100 x1876702068112384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.626914:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068112384/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.626921:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.626923:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626926:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766051.626931:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068112512:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766051.626945:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.626953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef4700 x1876702068112512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.626962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068112512/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.626969:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.626971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.626974:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068112512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766051.626978:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068112640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766051.626992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766051.627000:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766051/real 1789766051] req@ffff936010ef5180 x1876702068112640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766051.627009:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068112640/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766051.627016:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766051.627019:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766051.627021:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068112640:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766052.012944:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766053.034004:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766054.058775:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766055.082573:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766056.106051:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766056.745973:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.745982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766056.745986:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.745996:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746019:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746026:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766056.746073:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068112768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746104:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746107:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766056.746109:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746113:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746117:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746121:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746129:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766056.746140:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068112896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746157:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746160:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766056.746161:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746164:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746167:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746171:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746174:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766056.746182:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068113024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746190:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746193:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766056.746195:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746199:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746202:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746204:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746209:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766056.746219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068113152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746226:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746228:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766056.746230:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746233:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746235:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746238:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746243:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766056.746251:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068113280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746257:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746260:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766056.746262:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746265:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746267:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746270:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746274:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766056.746282:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068113408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766056.746290:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766056.746293:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766056.746295:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766056.746298:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3524 00000100:00080000:0.0:1789766056.746301:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766056.746309:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766056.746313:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766056.746322:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068113536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766056.748290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068112768:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766056.748359:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748426:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020dadf80 x1876702068112768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748445:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068112768/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748460:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748465:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068112768:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766056.748473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766056.748495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748509:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020dadc00 x1876702068112896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068112896/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748533:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748536:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068112896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766056.748543:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113024:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766056.748563:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020daf800 x1876702068113024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748586:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068113024/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748600:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748607:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766056.748613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113152:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766056.748635:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748646:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020daea00 x1876702068113152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748658:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068113152/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748666:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113152:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766056.748678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766056.748710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748722:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020dace00 x1876702068113280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068113280/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748742:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748748:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766056.748754:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068113408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766056.748774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020dae300 x1876702068113408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068113408/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748830:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748833:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748836:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068113408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766056.748842:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068113536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766056.748863:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766056.748873:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766056/real 1789766056] req@ffff936020dad180 x1876702068113536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766056.748882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068113536/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766056.748889:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766056.748892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766056.748894:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068113536:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766057.131245:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766058.154077:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766059.180151:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766060.204717:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766061.226060:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766061.866035:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.866044:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766061.866048:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.866057:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.866067:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.866073:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766061.866107:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068113664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766061.866169:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068113664:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766061.866261:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.866265:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766061.866267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.866272:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.866501:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.866507:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.866517:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766061.866529:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068113792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766061.870889:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113792:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766061.875986:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.875996:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766061.876000:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.876010:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.876020:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.876027:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.876039:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766061.876071:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068113920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766061.876097:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.876100:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766061.876102:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.876106:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.876109:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.876113:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.876117:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766061.876129:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068114048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766061.876137:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.876140:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766061.876141:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.876144:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.876147:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.876150:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.876154:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766061.876163:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068114176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766061.876171:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.876174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766061.876176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.876180:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.876183:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.876186:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.876190:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766061.876199:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068114304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766061.876207:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766061.876210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766061.876211:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766061.876215:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3529 00000100:00080000:3.0:1789766061.876218:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766061.876221:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766061.876225:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766061.876233:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068114432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766061.877827:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766061.878091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766061.878118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766061.878140:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068114304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766061.878161:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068114432:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766062.250185:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766063.275143:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766064.300397:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789766064.938293:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789766064.938335:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789766064.938350:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938390:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938399:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938406:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938413:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938420:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766064.938427:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789766064.938497:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa7100 x1876702068113920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938520:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068113920/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766064.938546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa4e00 x1876702068114048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938558:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068114048/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938565:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938569:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938572:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114048:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766064.938578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa4380 x1876702068114176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938589:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068114176/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938597:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766064.938608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa4700 x1876702068114304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938618:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068114304/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938625:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938629:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068114304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766064.938647:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa5180 x1876702068114432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068114432/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938665:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938672:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068114432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766064.938678:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa5500 x1876702068113792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938706:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068113792/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938713:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938717:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068113792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766064.938726:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766061/real 1789766064] req@ffff936010aa6a00 x1876702068113664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766064.938737:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068113664/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766064.938744:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766064.938747:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766064.938749:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068113664:192.168.203.137@tcp:250:kworker.0 00010000:00010000:1.0:1789766065.322041:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766066.346772:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766066.989288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766066.989302:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989311:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989322:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989329:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766066.989363:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068114560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989396:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766066.989398:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989402:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989406:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989409:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989419:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766066.989430:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068114688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989438:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989441:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766066.989443:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989446:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989449:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989457:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766066.989467:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068114816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989475:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989477:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766066.989479:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989483:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989486:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989489:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989493:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766066.989501:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068114944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989510:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989513:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766066.989514:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989517:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989520:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989523:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989526:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766066.989535:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068115072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989542:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989545:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766066.989546:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989550:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989553:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989556:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989559:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766066.989567:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068115200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766066.989575:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766066.989577:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766066.989579:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766066.989593:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3534 00000100:00080000:1.0:1789766066.989597:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766066.989600:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766066.989604:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766066.989613:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068115328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766066.994086:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068114560:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766066.994171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.994217:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef5c00 x1876702068114560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.994235:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068114560/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.994247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.994251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.994254:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068114560:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766066.994261:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766066.994283:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.994295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef6d80 x1876702068114688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.994306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068114688/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.994313:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.994316:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.994319:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766066.994324:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766066.994484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.994500:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef6300 x1876702068114816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.994512:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068114816/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.994520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.994524:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.994527:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766066.994533:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114944:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766066.994556:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.994566:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef4000 x1876702068114944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.995075:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068114944/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.995086:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.995090:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.995093:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068114944:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766066.995099:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766066.995126:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.995140:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef4380 x1876702068115072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.995150:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068115072/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.995157:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.995160:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.995163:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766066.995169:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068115200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766066.995188:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.995200:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef5880 x1876702068115200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.995210:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068115200/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.995222:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.995226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.995229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068115200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766066.995236:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068115328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766066.995258:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766066.995268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766066/real 1789766066] req@ffff936010ef4a80 x1876702068115328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766066.995279:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068115328/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766066.995286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766066.995289:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766066.995292:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068115328:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766067.373680:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766068.394018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766069.420873:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766070.442778:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766071.472043:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766072.107310:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107320:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766072.107324:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107333:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107344:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107366:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766072.107401:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068115456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107437:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107441:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766072.107443:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107446:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107450:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107453:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107462:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766072.107474:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068115584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107484:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107486:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766072.107488:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107492:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107495:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107498:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107502:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766072.107511:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068115712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107531:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107533:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766072.107535:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107538:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107541:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107545:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107549:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766072.107557:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068115840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107565:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107568:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766072.107570:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107573:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107575:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107589:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107593:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766072.107602:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068115968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107610:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107613:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766072.107615:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107618:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107621:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107624:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107629:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766072.107636:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068116096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766072.107645:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766072.107648:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766072.107649:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766072.107653:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3539 00000100:00080000:0.0:1789766072.107655:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766072.107659:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766072.107664:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766072.107673:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068116224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766072.107758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068115456:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766072.107837:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.107873:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020dac380 x1876702068115456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.107889:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068115456/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.107898:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.107902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.107905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068115456:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766072.107912:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766072.107935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.107947:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020dad880 x1876702068115584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.107958:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068115584/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.107964:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.107967:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.107969:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766072.107974:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766072.107992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.108002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020dac000 x1876702068115712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.108012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068115712/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.108018:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.108021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.108024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766072.108029:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115840:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766072.108047:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.108058:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020dae680 x1876702068115840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.108073:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068115840/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.108080:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.108082:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.108085:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115840:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766072.108091:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766072.108109:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.108119:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020daed80 x1876702068115968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.108129:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068115968/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.108136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.108139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.108142:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068115968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766072.108148:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766072.108168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.108179:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020daf480 x1876702068116096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.108188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068116096/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.108199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.108202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.108205:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766072.108209:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766072.108228:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766072.108238:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766072/real 1789766072] req@ffff936020dafb80 x1876702068116224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766072.108247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068116224/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766072.108254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766072.108256:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766072.108259:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116224:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766072.490046:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766073.516002:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766074.538037:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766075.562175:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766076.588004:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766077.225963:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.225972:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766077.225976:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.225986:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.225996:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226002:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766077.226038:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068116352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226067:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226071:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766077.226073:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226076:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226080:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226084:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226092:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766077.226104:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068116480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226115:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226118:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766077.226119:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226124:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226127:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226130:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226134:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766077.226144:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068116608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226152:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226155:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766077.226157:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226160:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226163:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226167:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226171:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766077.226183:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068116736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226191:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226194:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766077.226196:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226199:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226202:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226206:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226210:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766077.226219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068116864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226228:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226231:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766077.226233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226236:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226239:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226242:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226246:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766077.226256:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068116992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766077.226264:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766077.226266:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766077.226267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766077.226271:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3545 00000100:00080000:0.0:1789766077.226273:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766077.226276:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766077.226280:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766077.226288:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068117120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766077.226378:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068116352:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766077.226435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020dad500 x1876702068116352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226490:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068116352/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226499:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226507:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068116352:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766077.226514:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766077.226537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020dac700 x1876702068116480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226571:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068116480/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226578:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226581:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116480:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766077.226590:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766077.226610:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226621:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020daf100 x1876702068116608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226630:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068116608/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226637:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226641:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226644:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116608:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766077.226649:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116736:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766077.226668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020dafb80 x1876702068116736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226689:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068116736/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226697:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116736:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766077.226707:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766077.226724:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226735:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020daf480 x1876702068116864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226760:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068116864/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226769:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226772:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226775:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068116864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766077.226781:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766077.226829:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226841:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020daed80 x1876702068116992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068116992/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226861:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226864:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226867:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068116992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766077.226873:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068117120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766077.226893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766077.226903:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766077/real 1789766077] req@ffff936020dae680 x1876702068117120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766077.226914:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068117120/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766077.226920:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766077.226923:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766077.226926:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068117120:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766077.610296:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766078.634575:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766079.658495:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766080.685213:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766081.708038:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766082.349650:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.349669:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766082.349673:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.349683:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.349693:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.349700:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766082.349740:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068117248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766082.349769:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.349773:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766082.349775:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.349778:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.349782:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.349785:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766082.349794:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766082.349829:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068117248:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789766082.349864:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068117376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766082.349875:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.349878:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766082.349880:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.349884:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.349888:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.349893:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789766082.349894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789766082.349899:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766082.349913:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068117504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766082.349921:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.349924:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766082.349927:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.349930:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.349934:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.349938:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766082.349943:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa6680 x1876702068117248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789766082.349944:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766082.349958:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068117632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766082.349968:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.349971:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766082.349973:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.349977:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00100000:1.0:1789766082.349979:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068117248/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766082.349981:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.349988:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766082.349996:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789766082.350000:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350008:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766082.350010:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068117760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766082.350015:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068117248:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766082.350027:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.350031:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766082.350033:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766082.350036:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.350040:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.350043:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350048:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117376:192.168.203.137@tcp:38:kworker.0 00000008:00100000:3.0:1789766082.350048:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766082.350063:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068117888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766082.350073:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766082.350076:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766082.350080:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789766082.350081:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766082.350088:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3550 00000100:00080000:3.0:1789766082.350093:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766082.350097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766082.350101:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766082.350103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa7480 x1876702068117376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350117:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068117376/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350126:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789766082.350127:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068118016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766082.350134:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350142:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766082.350151:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766082.350175:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350189:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa5f80 x1876702068117504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350200:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068117504/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350223:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350226:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350229:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766082.350234:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117632:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766082.350254:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350265:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa4a80 x1876702068117632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350276:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068117632/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350284:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350286:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350289:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117632:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766082.350295:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766082.350312:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350341:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa5880 x1876702068117760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350353:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068117760/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350360:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350363:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350366:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068117760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766082.350385:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068117888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766082.350404:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350415:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa4000 x1876702068117888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350433:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068117888/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350453:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350457:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350460:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068117888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766082.350465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766082.350484:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766082.350495:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766082/real 1789766082] req@ffff936010aa7800 x1876702068118016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766082.350522:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068118016/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766082.350530:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766082.350532:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766082.350535:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118016:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766082.730042:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766083.756324:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766084.778066:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766085.802490:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766086.828059:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766087.466014:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.466024:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766087.466028:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.466037:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.466048:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.466067:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766087.466111:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068118144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766087.466190:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068118144:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766087.466263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766087.466294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.466297:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766087.466306:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.466311:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.466316:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.466320:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.466331:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766087.466357:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068118272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766087.468632:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef4a80 x1876702068118144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.468659:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068118144/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.468671:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.468675:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.468679:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068118144:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766087.468709:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766087.468757:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.468774:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef5880 x1876702068118272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.468786:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068118272/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.468795:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.468816:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.468819:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118272:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766087.471448:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.471457:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766087.471461:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.471480:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.471491:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.471499:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.471514:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766087.471548:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068118400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766087.471564:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.471567:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766087.471569:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.471573:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.471576:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.471580:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.471584:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766087.471594:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068118528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766087.471602:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.471605:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766087.471606:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.471609:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.471612:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.471615:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.471619:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766087.471628:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068118656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766087.471637:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.471640:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766087.471641:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.471644:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.471647:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.471650:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.471655:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766087.471664:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068118784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766087.471672:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766087.471675:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766087.471676:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766087.471679:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3555 00000100:00080000:1.0:1789766087.471682:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766087.471685:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766087.471688:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766087.471697:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068118912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766087.471792:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766087.471873:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.471904:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef4380 x1876702068118400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.471924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068118400/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.471934:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.471938:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.471942:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766087.471949:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118528:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766087.471970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.471981:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef4000 x1876702068118528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.471993:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068118528/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.472001:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.472004:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.472007:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118528:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766087.472013:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766087.472031:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.472041:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef6300 x1876702068118656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.472053:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068118656/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.472061:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.472064:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.472067:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068118656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766087.472073:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766087.472092:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.472103:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef6d80 x1876702068118784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.472114:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068118784/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.472121:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.472124:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.472127:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766087.472133:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766087.472151:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766087.472161:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766087/real 1789766087] req@ffff936010ef5c00 x1876702068118912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766087.472170:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068118912/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766087.472184:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766087.472187:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766087.472190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068118912:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766087.850022:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766088.874023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766089.898470:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766090.922048:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766091.946143:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766092.586789:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.586822:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766092.586827:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.586837:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.586859:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.586866:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766092.586902:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068119040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.586935:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.586939:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766092.586940:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.586944:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.586948:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.586952:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.586962:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766092.586974:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068119168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.586982:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.586986:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766092.586987:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.586991:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.586994:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.586997:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.587001:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766092.587012:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068119296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.587020:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.587024:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766092.587026:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.587029:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.587033:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.587036:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.587040:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766092.587049:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068119424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.587057:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.587060:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766092.587062:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.587065:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.587068:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.587071:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.587075:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766092.587084:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068119552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.587092:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.587095:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766092.587097:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.587100:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00100000:0.0:1789766092.587243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068119040:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766092.587302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.587354:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061df480 x1876702068119040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.587371:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068119040/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.587380:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.587383:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.587387:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068119040:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766092.587394:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766092.587435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.587448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061dd500 x1876702068119168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.587459:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068119168/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.587477:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.587481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.587484:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766092.587490:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766092.587509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.587520:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061dc000 x1876702068119296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.587531:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068119296/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.587538:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.587541:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.587544:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766092.587550:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119424:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766092.587569:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.587580:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061dce00 x1876702068119424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.587591:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068119424/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.587598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.587601:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.587604:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119424:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766092.587609:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766092.587628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.587638:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061dd180 x1876702068119552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.587650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068119552/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.587656:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.587659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.587662:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068119552:192.168.203.137@tcp:8:kworker.0 00000100:00080000:2.0:1789766092.587730:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.587735:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.587741:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766092.587754:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068119680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766092.587778:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766092.587781:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766092.587783:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766092.587787:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3560 00000100:00080000:2.0:1789766092.587791:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766092.587794:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766092.587819:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766092.587832:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068119808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766092.588512:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068119680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766092.588544:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.588557:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061df800 x1876702068119680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.588576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068119680/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.588584:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.588587:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.588590:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068119680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766092.588596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068119808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766092.588615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766092.588625:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766092/real 1789766092] req@ffff9360061dca80 x1876702068119808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766092.588642:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068119808/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766092.588650:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766092.588653:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766092.588656:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068119808:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766092.972191:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766093.994253:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766095.019167:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766096.042452:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766097.066018:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766097.707384:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707403:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766097.707407:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707418:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707430:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707437:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766097.707474:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068119936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707504:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707508:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766097.707510:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707514:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707518:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707521:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707531:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766097.707542:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068120064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707551:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707554:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766097.707556:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707560:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707562:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707566:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707570:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766097.707581:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068120192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707590:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707593:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766097.707595:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707598:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707601:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707604:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707608:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766097.707617:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068120320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707625:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707627:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766097.707629:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707632:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707635:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707637:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707642:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766097.707650:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068120448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707658:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707660:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766097.707662:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707664:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707667:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707670:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707673:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766097.707682:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068120576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766097.707690:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766097.707693:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766097.707694:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766097.707698:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3565 00000100:00080000:2.0:1789766097.707700:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766097.707703:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766097.707707:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766097.707716:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068120704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766097.707825:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068119936:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766097.707881:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.707919:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061de680 x1876702068119936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.707935:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068119936/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.707944:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.707948:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.707952:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068119936:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766097.707958:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766097.707981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.707993:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061dfb80 x1876702068120064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708004:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068120064/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708011:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708014:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708017:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766097.708022:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766097.708040:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.708050:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061dca80 x1876702068120192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708060:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068120192/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708068:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708071:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708074:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766097.708079:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120320:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766097.708099:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.708110:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061df800 x1876702068120320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708122:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068120320/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708130:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708133:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708136:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120320:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766097.708142:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766097.708161:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.708174:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061dd180 x1876702068120448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068120448/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708190:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708193:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708195:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766097.708200:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068120576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766097.708219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.708228:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061dce00 x1876702068120576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068120576/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708244:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708247:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708251:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068120576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766097.708256:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068120704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766097.708275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766097.708284:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766097/real 1789766097] req@ffff9360061dc000 x1876702068120704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766097.708294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068120704/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766097.708301:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766097.708303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766097.708306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068120704:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766098.093023:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766099.114423:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766100.138158:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766101.163778:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766102.186112:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766102.827240:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827250:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766102.827254:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827264:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827273:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827280:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766102.827316:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068120832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827360:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766102.827366:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827370:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827373:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827377:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827386:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766102.827397:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068120960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827406:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827409:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766102.827410:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827414:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827417:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827420:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827424:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766102.827441:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068121088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766102.827453:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827457:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827460:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827464:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827469:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766102.827480:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068121216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827489:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827492:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766102.827493:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827496:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827499:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827502:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827506:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766102.827515:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068121344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827523:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827526:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766102.827528:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827531:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827533:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827536:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827540:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766102.827549:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068121472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766102.827557:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766102.827560:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766102.827562:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766102.827565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3570 00000100:00080000:3.0:1789766102.827567:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766102.827570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766102.827574:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766102.827584:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068121600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766102.829145:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068120832:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766102.829202:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829242:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa6300 x1876702068120832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829260:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068120832/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829271:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829275:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829279:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068120832:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766102.829286:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766102.829307:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829319:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa7b80 x1876702068120960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829338:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068120960/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829363:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829366:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829369:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068120960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766102.829376:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766102.829399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa5c00 x1876702068121088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068121088/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829429:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829432:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829434:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766102.829440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121216:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766102.829457:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829467:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa7800 x1876702068121216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829477:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068121216/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829490:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829493:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829496:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121216:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766102.829502:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121344:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766102.829520:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829531:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa4000 x1876702068121344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829543:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068121344/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829552:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829555:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829558:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766102.829564:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068121472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766102.829583:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829594:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa5880 x1876702068121472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829606:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068121472/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829616:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829619:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068121472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766102.829625:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068121600:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766102.829643:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766102.829654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766102/real 1789766102] req@ffff936010aa4a80 x1876702068121600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766102.829665:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068121600/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766102.829672:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766102.829675:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766102.829678:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068121600:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766103.210397:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766104.235545:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766105.258991:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766106.288385:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766107.306101:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766107.946659:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.946668:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766107.946672:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.946682:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.946693:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.946700:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766107.946738:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068121728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.946768:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.946772:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766107.946774:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.946778:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.946781:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.946785:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.946793:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766107.946830:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068121856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.946840:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.946843:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766107.946845:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.946849:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.946852:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.946856:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.946861:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766107.946873:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068121984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.946881:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.946884:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766107.946886:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.946889:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.946892:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.946896:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.946900:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766107.946909:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068122112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.946917:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.946920:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766107.946921:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.946924:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.946927:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.946931:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.947076:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766107.947090:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068122240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.947098:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.947102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766107.947114:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.947117:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.947122:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.947126:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.947131:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766107.947141:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068122368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766107.947149:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766107.947151:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766107.947153:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766107.947156:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3575 00000100:00080000:2.0:1789766107.947158:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766107.947161:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766107.947164:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766107.947173:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068122496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766107.948613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068121728:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766107.948688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.948733:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061dd500 x1876702068121728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.948750:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068121728/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.948761:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.948765:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.948769:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068121728:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766107.948777:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766107.948796:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.948834:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061df480 x1876702068121856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949049:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068121856/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949058:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949061:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949065:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766107.949071:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766107.949093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.949105:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061df100 x1876702068121984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949117:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068121984/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949126:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949129:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949133:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068121984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766107.949139:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122112:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766107.949681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.949698:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061dd880 x1876702068122112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949713:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068122112/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949721:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949725:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949728:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122112:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766107.949735:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766107.949753:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.949763:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061ddc00 x1876702068122240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949774:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068122240/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949781:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949788:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766107.949793:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068122368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766107.949833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.949856:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061de300 x1876702068122368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949867:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068122368/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949881:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949884:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949887:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068122368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766107.949893:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068122496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766107.949909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766107.949918:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766107/real 1789766107] req@ffff9360061dc380 x1876702068122496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766107.949929:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068122496/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766107.949937:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766107.949940:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766107.949943:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068122496:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766108.330773:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766109.354098:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766110.380143:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766111.402511:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766112.426130:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766113.067212:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.067221:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766113.067226:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.067236:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.067247:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.067256:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766113.067551:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068122624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766113.067634:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068122624:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766113.067701:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766113.067867:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.067871:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766113.067873:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.067878:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.067882:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.067886:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.067895:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766113.067909:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068122752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766113.068006:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa5f80 x1876702068122624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.068024:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068122624/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.068038:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.068042:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.068045:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068122624:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766113.068077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766113.068100:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.068113:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa7480 x1876702068122752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.068124:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068122752/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.068133:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.068137:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.068141:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122752:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766113.069089:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.069095:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766113.069097:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.069103:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.069108:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.069113:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.069121:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766113.069138:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068122880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766113.069149:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.069152:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766113.069153:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.069156:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.069159:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.069163:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.069167:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766113.069175:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068123008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766113.069184:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.069187:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766113.069188:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.069192:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.069196:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.069199:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.069204:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766113.069213:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068123136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766113.069222:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766113.069225:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.069228:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766113.069230:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.069234:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.069237:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.069240:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.069244:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766113.069328:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068123264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766113.069337:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766113.069340:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766113.069342:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766113.069345:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3580 00000100:00080000:3.0:1789766113.069349:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766113.069354:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766113.069360:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766113.069372:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068123392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766113.069388:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.069409:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa6680 x1876702068122880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.069425:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068122880/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.069435:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.069438:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.069441:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068122880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766113.069448:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123008:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766113.069468:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.069479:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa6a00 x1876702068123008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.069490:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068123008/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.069498:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.069501:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.069503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123008:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766113.069518:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766113.070157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.070171:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa5500 x1876702068123136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.070183:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068123136/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.070191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.070195:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.070198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766113.070204:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068123264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766113.070220:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.070230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa5180 x1876702068123264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.070241:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068123264/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.070248:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.070251:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.070254:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068123264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766113.070259:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068123392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766113.070287:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766113.070298:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766113/real 1789766113] req@ffff936010aa4700 x1876702068123392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766113.070310:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068123392/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766113.070317:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766113.070319:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766113.070322:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068123392:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766113.450534:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766114.474702:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766115.498151:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766116.525818:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766117.546083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766118.186023:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186034:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766118.186038:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766118.186047:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186058:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186065:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766118.186101:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068123520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766118.186183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068123520:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766118.186245:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766118.186274:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186278:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766118.186280:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766118.186284:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186288:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186292:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766118.186301:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766118.186314:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068123648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766118.186443:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186447:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766118.186448:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766118.186453:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186456:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186460:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766118.186464:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766118.186476:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068123776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766118.186480:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa4380 x1876702068123520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766118.186486:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186489:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766118.186490:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766118.186494:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186497:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186500:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068123520/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789766118.186504:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789766118.186511:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789766118.186513:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068123904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766118.186515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186519:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068123520:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766118.186521:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186524:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766118.186525:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766118.186528:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186530:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186533:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766118.186536:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766118.186538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766118.186544:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068124032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766118.186552:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766118.186555:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766118.186556:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789766118.186559:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766118.186560:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00080000:3.0:1789766118.186562:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766118.186565:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186569:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa4e00 x1876702068123648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789766118.186569:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766118.186577:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068124160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766118.186580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068123648/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766118.186586:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766118.186588:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766118.186588:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766118.186590:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766118.186591:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789766118.186593:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3586 00000100:00100000:0.0:1789766118.186594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123648:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766118.186596:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766118.186598:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123776:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766118.186599:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766118.186603:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766118.186610:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068124288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766118.186615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186623:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa7100 x1876702068123776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766118.186633:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068123776/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766118.186644:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766118.186647:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186650:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766118.186656:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123904:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766118.186676:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186685:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa4700 x1876702068123904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766118.186695:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068123904/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766118.186702:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766118.186705:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186707:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068123904:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766118.186721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766118.186735:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186743:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa5180 x1876702068124032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766118.186752:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068124032/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766118.186758:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766118.186761:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186763:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766118.186768:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068124160:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766118.186780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186787:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa5500 x1876702068124160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766118.186796:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068124160/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766118.186825:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766118.186828:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186831:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068124160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766118.186835:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068124288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766118.186850:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766118.186858:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766118/real 1789766118] req@ffff936010aa6a00 x1876702068124288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766118.186868:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068124288/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766118.186875:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766118.186878:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766118.186880:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068124288:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766118.570098:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766119.594622:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766120.618237:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766121.642258:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766122.671198:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766123.306243:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306254:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766123.306258:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306267:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306277:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766123.306284:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766123.306320:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068124416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766123.306359:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306363:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766123.306365:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306369:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306372:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766123.306375:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306384:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766123.306396:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068124544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766123.306404:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306408:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766123.306410:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306414:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306417:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766123.306421:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068124416:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766123.306424:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306432:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766123.306442:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068124672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766123.306452:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306456:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766123.306457:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306461:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306466:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766123.306469:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306473:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766123.306483:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068124800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766123.306489:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766123.306497:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306501:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766123.306503:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306507:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306511:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766123.306514:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306519:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766123.306529:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068124928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766123.306537:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306540:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766123.306542:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306546:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306550:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766123.306553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa6680 x1876702068124416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766123.306557:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306565:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766123.306575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068124416/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.306589:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789766123.306594:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068125056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766123.306596:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.306603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068124416:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766123.306609:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766123.306628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766123.306637:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa7480 x1876702068124544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766123.306648:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068124544/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.306655:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.306659:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.306670:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766123.306696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766123.306716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766123.306727:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa5f80 x1876702068124672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766123.306739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068124672/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.306746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.306749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.306752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766123.306758:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124800:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766123.306789:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766123.306839:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa4a80 x1876702068124800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766123.306851:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068124800/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.306859:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.306863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.306866:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124800:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766123.306872:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766123.306894:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766123.306902:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766123.306907:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789766123.306911:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa5880 x1876702068124928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766123.306911:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766123.306919:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3591 00000100:00080000:3.0:1789766123.306927:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766123.306934:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068124928/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766123.306935:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766123.306943:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766123.306956:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068125184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766123.307230:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.307234:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.307238:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068124928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766123.307257:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125056:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766123.307279:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766123.307290:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa4000 x1876702068125056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766123.307300:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068125056/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.307307:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.307310:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.307313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125056:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766123.307318:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766123.307337:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766123.307348:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766123/real 1789766123] req@ffff936010aa7800 x1876702068125184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766123.307359:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068125184/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766123.307366:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766123.307369:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766123.307372:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125184:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766123.690796:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766124.714010:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766125.738084:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766126.762426:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766127.786494:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766128.426187:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.426197:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766128.426201:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.426210:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.426222:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.426296:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766128.426338:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068125312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766128.426407:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068125312:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766128.426598:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.426603:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766128.426605:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.426609:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.426624:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.426629:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766128.426638:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766128.426652:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068125440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766128.432194:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125440:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766128.432735:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.432741:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766128.432745:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.432750:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.432756:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.432764:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766128.433538:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766128.433573:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068125568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766128.433587:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.433590:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766128.433592:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.433596:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.433602:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.433607:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766128.433610:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766128.433928:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766128.434489:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068125696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766128.434505:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.434509:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766128.434511:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.434515:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.434520:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.434525:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766128.434531:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766128.434542:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068125824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766128.434551:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.434554:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766128.434556:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.434559:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.434562:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.434565:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766128.434570:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766128.434579:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068125952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766128.434595:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766128.434598:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766128.434599:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766128.434603:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3596 00000100:00080000:2.0:1789766128.434606:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766128.434609:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766128.434906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766128.434940:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766128.435744:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125952:192.168.203.137@tcp:38:kworker.0 00000008:00100000:2.0:1789766128.436410:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766128.436439:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068126080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766128.436503:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126080:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766128.810928:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766129.834026:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766130.858009:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789766131.500998:0:2382:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789766131.501048:0:2382:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789766131.501067:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501109:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501118:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501126:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501133:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501151:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766131.501160:0:2382:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789766131.501199:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061df100 x1876702068126080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501238:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068126080/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501250:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501255:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501259:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766131.501267:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061de300 x1876702068125696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501281:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068125696/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501289:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501292:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501295:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125696:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766131.501301:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061ddc00 x1876702068125824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501311:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068125824/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501317:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501320:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501323:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766131.501329:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061dd880 x1876702068125952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501340:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068125952/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501347:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501350:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501353:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068125952:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766131.501359:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061dc380 x1876702068125568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501370:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068125568/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501378:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501381:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501384:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125568:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766131.501390:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061ddf80 x1876702068125440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501401:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068125440/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501413:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068125440:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766131.501420:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766128/real 1789766131] req@ffff9360061dc700 x1876702068125312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766131.501432:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068125312/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766131.501438:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766131.501441:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766131.501443:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068125312:192.168.203.137@tcp:250:kworker.0 00010000:00010000:0.0:1789766131.883517:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766132.907054:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766133.546248:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.546256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766133.546260:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548218:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548233:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548245:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766133.548291:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068126208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548327:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548332:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766133.548334:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548339:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548343:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548347:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548357:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766133.548369:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068126336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548378:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548381:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766133.548382:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548386:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548389:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548392:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548395:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766133.548406:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068126464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548413:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548415:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766133.548417:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548420:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548435:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548438:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548443:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766133.548452:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068126592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548461:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548463:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766133.548465:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548468:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548470:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548473:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766133.548484:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068126720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548491:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548494:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766133.548496:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548499:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548502:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548633:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548639:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766133.548650:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068126848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766133.548658:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766133.548661:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766133.548663:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766133.548666:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3601 00000100:00080000:2.0:1789766133.548670:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766133.548673:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766133.548676:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766133.548684:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068126976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766133.552876:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068126208:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766133.552950:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.552992:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061df480 x1876702068126208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553010:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068126208/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553022:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553026:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553029:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068126208:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766133.553037:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766133.553061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553075:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061dd500 x1876702068126336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553087:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068126336/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553102:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553105:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553108:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126336:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766133.553137:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766133.553158:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553170:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061dc000 x1876702068126464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553182:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068126464/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553195:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766133.553205:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126592:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766133.553225:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553236:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061dce00 x1876702068126592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553246:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068126592/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553257:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553260:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126592:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766133.553267:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766133.553287:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553298:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061dd180 x1876702068126720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553308:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068126720/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553321:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068126720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766133.553327:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126848:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766133.553346:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061df800 x1876702068126848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068126848/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553375:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553378:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553381:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126848:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766133.553387:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766133.553406:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766133.553416:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766133/real 1789766133] req@ffff9360061dca80 x1876702068126976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766133.553438:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068126976/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766133.553446:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766133.553449:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766133.553452:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068126976:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766133.930550:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766134.955099:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766135.978522:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766137.002208:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766138.027072:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766138.666324:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766138.666339:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666348:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666360:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666370:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766138.666408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068127104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666442:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666446:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766138.666448:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666453:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666457:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666460:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666471:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766138.666483:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068127232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666492:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666496:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766138.666497:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666501:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666504:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666508:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666512:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766138.666521:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068127360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666529:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666532:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766138.666533:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666537:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666541:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666544:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666548:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766138.666572:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068127488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666581:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766138.666586:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666589:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666592:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666597:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666601:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766138.666611:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068127616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666620:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666623:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766138.666624:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666627:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666631:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666634:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666639:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766138.666648:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068127744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766138.666657:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766138.666660:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766138.666662:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766138.666665:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3606 00000100:00080000:3.0:1789766138.666668:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766138.666672:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766138.666676:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766138.666686:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068127872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766138.668855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068127104:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766138.668927:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669459:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa5c00 x1876702068127104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669484:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068127104/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669495:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669499:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669503:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068127104:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766138.669510:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766138.669537:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669549:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa7b80 x1876702068127232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068127232/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669569:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669572:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127232:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766138.669581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766138.669600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa6300 x1876702068127360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068127360/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669628:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669631:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669634:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766138.669639:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127488:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766138.669657:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669666:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa7800 x1876702068127488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669677:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068127488/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127488:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766138.669696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766138.669712:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa4000 x1876702068127616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669743:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068127616/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669751:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669754:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068127616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766138.669762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068127744:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766138.669781:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669791:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa5880 x1876702068127744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669850:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068127744/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669857:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669860:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669863:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068127744:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766138.669869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068127872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766138.669888:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766138.669899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766138/real 1789766138] req@ffff936010aa4a80 x1876702068127872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766138.669911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068127872/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766138.669918:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766138.669921:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766138.669924:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068127872:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766139.050474:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766140.073985:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766141.098413:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766142.122038:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766143.146164:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766143.786090:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786100:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766143.786104:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786110:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786120:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786126:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766143.786157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068128000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766143.786222:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068128000:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766143.786268:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789766143.786295:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766143.786299:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786302:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786304:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786307:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766143.786325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068128128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766143.786383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dac000 x1876702068128000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786395:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068128000/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786406:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786409:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068128000:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766143.786427:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766143.786443:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786451:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dad880 x1876702068128128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068128128/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786469:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786471:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786472:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128128:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789766143.786524:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786527:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766143.786528:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786531:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786535:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786538:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786542:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766143.786550:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068128256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766143.786556:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766143.786561:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786564:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786566:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786569:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786571:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766143.786577:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068128384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766143.786582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786584:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766143.786586:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786588:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786590:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786592:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786595:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766143.786600:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068128512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766143.786604:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786606:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766143.786607:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786609:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786611:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786614:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786617:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766143.786622:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068128640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766143.786628:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766143.786630:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766143.786631:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766143.786632:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3611 00000100:00080000:0.0:1789766143.786634:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766143.786635:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766143.786637:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766143.786643:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068128768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766143.786693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766143.786711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dac380 x1876702068128256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786726:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068128256/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786732:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786735:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786737:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128256:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766143.786740:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128384:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766143.786752:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786758:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dad180 x1876702068128384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068128384/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786777:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128384:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766143.786781:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766143.786792:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786818:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dae300 x1876702068128512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786826:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068128512/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786833:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786834:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786836:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068128512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766143.786840:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068128640:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766143.786855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786862:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020dace00 x1876702068128640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786868:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068128640/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786873:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786876:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068128640:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766143.786882:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068128768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766143.786893:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766143.786899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766143/real 1789766143] req@ffff936020daea00 x1876702068128768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766143.786908:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068128768/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766143.786912:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766143.786914:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766143.786916:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068128768:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766144.171372:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766145.200823:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766146.218573:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766147.242068:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766148.267468:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766148.906109:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766148.906124:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766148.906134:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00080000:0.0:1789766148.906144:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906153:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766148.906187:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068128896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766148.906219:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906223:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766148.906226:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766148.906230:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00080000:0.0:1789766148.906234:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906239:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766148.906258:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766148.906272:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068129024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766148.906284:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906288:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766148.906290:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766148.906293:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068128896:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789766148.906296:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00080000:0.0:1789766148.906299:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906303:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766148.906310:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766148.906324:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068129152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766148.906335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766148.906340:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766148.906344:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00080000:0.0:1789766148.906356:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906362:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:1.0:1789766148.906366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:0.0:1789766148.906369:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766148.906384:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068129280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766148.906393:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906397:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766148.906399:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766148.906403:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00080000:0.0:1789766148.906408:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789766148.906408:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020daf800 x1876702068128896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766148.906416:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766148.906425:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766148.906440:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068129408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766148.906443:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068128896/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766148.906461:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766148.906465:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766148.906466:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766148.906471:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766148.906473:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:0.0:1789766148.906477:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3616 00000100:00100000:1.0:1789766148.906480:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068128896:192.168.203.137@tcp:250:kworker.0 00000100:00080000:0.0:1789766148.906483:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906490:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766148.906495:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766148.906506:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068129536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766148.906509:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129024:192.168.203.137@tcp:38:kworker.0 00000100:00080000:0.0:1789766148.906526:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766148.906530:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766148.906533:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766148.906537:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3616 00000800:00000100:1.0:1789766148.906542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789766148.906543:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766148.906550:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766148.906557:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020dadc00 x1876702068129024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789766148.906568:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766148.906573:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068129024/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766148.906583:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068129664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766148.906584:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.906592:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.906599:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766148.906606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766148.906627:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766148.906639:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020dadf80 x1876702068129152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766148.906650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068129152/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766148.906657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.906660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.906663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129152:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766148.906669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129280:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766148.906685:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766148.906845:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020daea00 x1876702068129280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766148.906862:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068129280/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766148.906870:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.906873:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.906876:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129280:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766148.906881:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766148.906903:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766148.906914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020dace00 x1876702068129408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766148.906924:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068129408/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766148.906932:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.906935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.906938:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766148.906955:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068129536:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766148.906974:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766148.906984:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020dae300 x1876702068129536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766148.906994:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068129536/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766148.907007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.907010:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.907024:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068129536:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766148.907030:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068129664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766148.907049:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766148.907060:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766148/real 1789766148] req@ffff936020dad180 x1876702068129664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766148.907070:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068129664/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766148.907078:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766148.907081:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766148.907084:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068129664:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766149.290081:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766150.314147:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766151.338083:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766152.362707:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766153.386138:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766154.026006:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766154.026015:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766154.026019:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026029:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026038:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766154.026045:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766154.026079:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068129792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766154.026104:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789766154.026198:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068129792:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766154.026292:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766154.026347:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa5f80 x1876702068129792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766154.026369:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068129792/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.026392:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.026398:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.026408:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068129792:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766154.026410:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766154.026417:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026424:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026430:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766154.026437:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026446:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766154.026463:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068129920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766154.026483:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766154.026487:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766154.026488:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026491:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026495:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766154.026498:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026502:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766154.026511:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068130048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766154.026519:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766154.026521:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766154.026523:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026526:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026530:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766154.026534:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026539:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766154.026547:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129920:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766154.026554:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068130176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766154.026567:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766154.026570:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766154.026572:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026577:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3621 00000800:00000100:1.0:1789766154.026579:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766154.026592:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789766154.026603:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa7480 x1876702068129920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766154.026603:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026614:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766154.026631:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068129920/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766154.026637:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068130304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766154.026658:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766154.026663:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766154.026669:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789766154.026677:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789766154.026682:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068129920:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766154.026682:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766154.026695:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026700:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789766154.026701:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130048:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766154.026709:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026718:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766154.026738:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068130432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789766154.026745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766154.026760:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766154.026764:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766154.026767:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766154.026773:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa6680 x1876702068130048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766154.026778:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3621 00000100:00080000:3.0:1789766154.026790:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766154.026830:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766154.026835:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766154.026848:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068130560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766154.026866:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068130048/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.026878:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.026884:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.026889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766154.026908:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130176:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766154.026932:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766154.026945:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa6a00 x1876702068130176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766154.026957:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068130176/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.026965:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.026968:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.026984:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130176:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766154.026991:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766154.027011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766154.027022:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa5500 x1876702068130304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766154.027032:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068130304/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.027040:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.027043:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.027046:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766154.027053:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068130432:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766154.027072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766154.027083:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa5180 x1876702068130432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766154.027094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068130432/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.027103:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.027106:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.027109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068130432:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766154.027115:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068130560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766154.027133:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766154.027144:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766154/real 1789766154] req@ffff936010aa4700 x1876702068130560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766154.027155:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068130560/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766154.027162:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766154.027165:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766154.027168:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068130560:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766154.412019:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766155.434701:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766156.458226:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766157.482000:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766158.507071:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766159.145922:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.145933:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766159.145944:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.145954:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.145964:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.145972:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766159.146011:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068130688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146043:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146047:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766159.146049:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146052:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146056:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146059:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146068:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766159.146079:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068130816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146087:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146090:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766159.146091:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146095:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146097:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146101:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146105:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766159.146115:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068130944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146123:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146126:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766159.146127:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146130:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146133:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146136:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766159.146151:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068131072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146159:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146162:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766159.146163:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146166:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146169:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146172:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146176:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766159.146184:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068131200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146194:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766159.146195:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146198:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146201:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146204:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146208:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766159.146217:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068131328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766159.146225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766159.146228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766159.146229:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766159.146232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3626 00000100:00080000:3.0:1789766159.146234:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766159.146237:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766159.146240:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766159.146249:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068131456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766159.147188:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068130688:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766159.147249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147288:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa7100 x1876702068130688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147306:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068130688/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147316:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147321:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147325:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068130688:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766159.147332:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766159.147372:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147386:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa4e00 x1876702068130816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068130816/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147404:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147407:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147410:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130816:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766159.147415:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766159.147435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa4380 x1876702068130944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147455:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068130944/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147464:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147467:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068130944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766159.147473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131072:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766159.147491:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147501:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa4700 x1876702068131072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068131072/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147518:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147521:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147523:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131072:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766159.147529:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766159.147546:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa5180 x1876702068131200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147567:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068131200/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147575:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147578:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766159.147584:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068131328:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766159.147600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147610:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa5500 x1876702068131328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147629:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068131328/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147636:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147638:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147641:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068131328:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766159.147647:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068131456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766159.147663:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766159.147671:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766159/real 1789766159] req@ffff936010aa6a00 x1876702068131456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766159.147681:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068131456/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766159.147689:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766159.147691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766159.147695:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068131456:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766159.530043:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766160.554025:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766161.578968:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766162.603276:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766163.629018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766164.265973:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.265984:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766164.265988:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.265998:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266009:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266016:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766164.266053:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068131584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266085:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266088:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766164.266090:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266095:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266098:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266110:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266119:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766164.266132:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068131712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266141:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266144:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766164.266146:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266149:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266153:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266157:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266161:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766164.266172:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068131840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266181:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266184:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766164.266185:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266189:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266192:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266195:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266199:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766164.266208:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068131968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266217:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266220:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766164.266221:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266225:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266228:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266231:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266235:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766164.266245:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068132096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266253:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266256:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766164.266258:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266261:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266264:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266268:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266272:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766164.266280:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068132224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766164.266288:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766164.266290:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766164.266292:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766164.266295:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3632 00000100:00080000:2.0:1789766164.266298:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766164.266301:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766164.266305:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766164.266313:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068132352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766164.266450:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068131584:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766164.266512:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266550:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061dfb80 x1876702068131584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266566:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068131584/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266587:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266590:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266594:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068131584:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766164.266607:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766164.266628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266639:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061de680 x1876702068131712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068131712/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266664:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131712:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766164.266669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766164.266686:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266696:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061dca80 x1876702068131840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266707:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068131840/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266714:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266717:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266720:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766164.266725:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131968:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766164.266742:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266750:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061df800 x1876702068131968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068131968/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266766:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266769:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266772:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068131968:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766164.266778:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766164.266792:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266842:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061dd180 x1876702068132096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068132096/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266859:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266864:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766164.266869:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068132224:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766164.266884:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266894:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061dce00 x1876702068132224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068132224/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266928:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068132224:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766164.266933:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068132352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766164.266951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766164.266960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766164/real 1789766164] req@ffff9360061dc000 x1876702068132352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766164.266970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068132352/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766164.266977:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766164.266980:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766164.266983:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068132352:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766164.650468:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766165.674989:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766166.698176:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766167.723320:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766168.746014:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766169.390161:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.390171:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766169.390175:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.390183:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.390193:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.390200:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766169.390237:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068132480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766169.390332:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068132480:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766169.390580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766169.390609:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.390613:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766169.390615:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.390620:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.390624:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.390628:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.390638:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766169.390652:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068132608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766169.392218:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa6680 x1876702068132480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.392242:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068132480/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.392252:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.392256:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.392260:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068132480:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766169.392386:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766169.392423:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.392436:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa7480 x1876702068132608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.392448:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068132608/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.392456:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.392459:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.392462:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132608:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766169.394234:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.394244:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766169.394248:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.394257:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.394267:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.394274:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.394285:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766169.394585:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068132736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766169.394600:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.394603:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766169.394605:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.394608:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.394613:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.394617:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.394623:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766169.394634:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068132864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766169.394642:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.394645:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766169.394647:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.394650:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.394653:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.394656:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.394660:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766169.394669:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068132992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766169.394676:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.394679:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766169.394681:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.394684:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.394687:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.394691:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.394695:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766169.394707:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068133120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766169.394720:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766169.394723:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766169.394725:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766169.394728:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3637 00000100:00080000:3.0:1789766169.394731:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766169.394734:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766169.394739:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766169.394748:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068133248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766169.394846:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132736:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766169.394909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.394932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa5f80 x1876702068132736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.394947:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068132736/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.394956:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.394960:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.394963:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766169.394970:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132864:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766169.394991:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.395002:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa4a80 x1876702068132864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.395013:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068132864/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.395020:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.395023:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.395026:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132864:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766169.395032:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132992:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766169.395052:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.395062:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa5880 x1876702068132992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.399461:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068132992/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.399473:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.399478:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.399483:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068132992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766169.399494:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068133120:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766169.399562:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.399584:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa4000 x1876702068133120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.399597:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068133120/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.399606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.399609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.399612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068133120:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766169.399619:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068133248:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766169.399639:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766169.399650:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766169/real 1789766169] req@ffff936010aa7800 x1876702068133248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766169.399661:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068133248/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766169.399669:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766169.399672:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766169.399675:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068133248:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766169.770456:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766170.796161:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766171.818017:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766172.841975:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766173.866085:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766174.512247:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512256:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766174.512260:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512269:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512280:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512287:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766174.512322:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068133376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512350:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512354:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766174.512356:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512359:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512363:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512367:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512375:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766174.512387:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068133504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512449:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512452:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766174.512454:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512458:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512461:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512465:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512470:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766174.512482:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068133632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512491:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512494:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766174.512495:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512499:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512502:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512506:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512510:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766174.512520:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068133760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512528:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512531:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766174.512568:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512573:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512576:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512579:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512658:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766174.512671:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068133888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512679:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512682:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766174.512684:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512687:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512690:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512694:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512698:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766174.512707:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068134016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766174.512763:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766174.512766:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766174.512767:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766174.512772:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3642 00000100:00080000:1.0:1789766174.512775:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766174.512779:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766174.512784:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766174.512872:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068134144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766174.513317:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068133376:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766174.513414:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef5180 x1876702068133376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513516:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068133376/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513529:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068133376:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766174.513557:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766174.513582:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513595:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef4700 x1876702068133504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513607:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068133504/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133504:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766174.513628:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766174.513650:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513667:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef7100 x1876702068133632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068133632/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513702:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513706:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766174.513716:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133760:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766174.513738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513749:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef4e00 x1876702068133760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513761:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068133760/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513769:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133760:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766174.513787:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766174.513876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513887:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef7b80 x1876702068133888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513898:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068133888/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513906:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513909:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513912:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068133888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766174.513919:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134016:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766174.513940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.513952:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef7480 x1876702068134016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.513964:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068134016/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.513971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.513974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.513977:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134016:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766174.513982:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766174.514249:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766174.514261:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766174/real 1789766174] req@ffff936010ef6680 x1876702068134144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766174.514273:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068134144/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766174.514281:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766174.514284:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766174.514287:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134144:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766174.890040:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766175.914068:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766176.938172:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766177.962130:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766178.989648:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766179.625972:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.625983:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766179.625988:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.625997:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00080000:2.0:1789766179.626008:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626014:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766179.626051:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068134272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766179.626087:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626090:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766179.626092:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626096:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00080000:2.0:1789766179.626099:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626102:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766179.626109:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766179.626120:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068134400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766179.626129:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626133:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766179.626134:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626138:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00080000:2.0:1789766179.626142:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766179.626144:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068134272:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766179.626149:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766179.626154:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766179.626164:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068134528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766179.626174:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766179.626178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626180:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00080000:2.0:1789766179.626184:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626187:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766179.626191:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766179.626200:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068134656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766179.626205:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789766179.626214:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626217:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766179.626219:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626223:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00080000:2.0:1789766179.626225:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626228:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766179.626232:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766179.626242:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068134784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766179.626248:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061dd500 x1876702068134272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766179.626254:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626263:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766179.626267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626272:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3647 00000100:00100000:0.0:1789766179.626277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068134272/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766179.626278:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626286:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:0.0:1789766179.626295:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789766179.626295:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789766179.626303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626311:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068134272:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789766179.626311:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068134912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766179.626323:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134400:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766179.626338:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766179.626340:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766179.626342:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766179.626347:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3647 00000800:00000100:0.0:1789766179.626352:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789766179.626353:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766179.626359:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766179.626364:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766179.626368:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061df480 x1876702068134400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766179.626378:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068135040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766179.626384:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068134400/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626416:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134400:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766179.626438:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134528:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766179.626453:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766179.626461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061dc700 x1876702068134528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766179.626471:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068134528/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626479:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626481:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626484:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766179.626489:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134656:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766179.626503:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766179.626511:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061ddf80 x1876702068134656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766179.626521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068134656/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626528:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626531:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626534:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134656:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766179.626538:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766179.626551:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766179.626558:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061dc380 x1876702068134784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766179.626567:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068134784/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626579:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626582:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626585:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068134784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766179.626589:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134912:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766179.626605:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766179.626614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061dd880 x1876702068134912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766179.626624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068134912/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068134912:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766179.626641:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766179.626655:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766179.626663:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766179/real 1789766179] req@ffff9360061ddc00 x1876702068135040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766179.626672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068135040/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766179.626679:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766179.626681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766179.626685:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135040:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766180.012247:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766181.034394:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766182.058445:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766183.081963:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766184.106188:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766184.746267:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746277:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766184.746281:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746290:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746300:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746307:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766184.746341:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068135168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746370:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746374:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766184.746376:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746380:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746383:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746387:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746396:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766184.746408:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068135296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746432:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746437:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766184.746439:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746443:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746446:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746449:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746453:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766184.746463:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068135424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746470:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746474:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766184.746475:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746479:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746481:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746484:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746488:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766184.746497:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068135552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746504:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746507:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766184.746509:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746512:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746514:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746517:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746521:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766184.746529:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068135680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746536:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746538:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766184.746539:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746542:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746545:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746548:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746552:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766184.746559:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068135808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766184.746566:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766184.746569:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766184.746570:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766184.746573:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3652 00000100:00080000:3.0:1789766184.746575:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766184.746578:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766184.746581:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766184.746588:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068135936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766184.746883:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068135168:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766184.746940:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.746998:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa6300 x1876702068135168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747016:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068135168/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747024:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747028:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747031:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068135168:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766184.747038:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766184.747061:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747072:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa7b80 x1876702068135296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747083:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068135296/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747090:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747092:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747095:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135296:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766184.747101:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766184.747119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa5c00 x1876702068135424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747140:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068135424/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747148:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747151:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747154:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766184.747159:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135552:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766184.747177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747349:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa7800 x1876702068135552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068135552/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747376:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135552:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766184.747381:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766184.747400:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747410:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa4000 x1876702068135680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747704:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068135680/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747711:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747714:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068135680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766184.747723:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135808:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766184.747745:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747767:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa5880 x1876702068135808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747778:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068135808/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747791:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135808:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766184.747796:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766184.747855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766184.747866:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766184/real 1789766184] req@ffff936010aa4a80 x1876702068135936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766184.747876:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068135936/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766184.747891:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766184.747894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766184.747897:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068135936:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766185.134334:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766186.159286:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766187.178665:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766188.203953:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766189.226006:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766189.866545:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766189.866559:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866568:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866578:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866598:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766189.866635:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068136064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.866665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866669:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766189.866671:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866675:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866679:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866682:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.866692:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766189.866705:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068136192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.866725:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866728:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766189.866730:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866734:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866737:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866741:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.866745:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766189.866756:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068136320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.866764:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866767:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766189.866769:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866772:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866776:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866779:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.866783:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766189.866793:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068136448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.866831:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866834:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766189.866836:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866839:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866842:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866846:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.866851:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766189.866861:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068136576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.866870:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.866884:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766189.866886:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.866890:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.866893:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.866897:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.866901:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766189.867173:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068136064:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766189.867232:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.867270:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa5f80 x1876702068136064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.867287:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068136064/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.867298:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.867302:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.867305:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068136064:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766189.867312:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766189.867336:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.867350:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa7480 x1876702068136192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.867362:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068136192/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.867375:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.867378:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.867382:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136192:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766189.867401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766189.867420:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.867431:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa6680 x1876702068136320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.867442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068136320/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.867450:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.867453:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.867456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766189.867462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136448:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766189.867481:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.867492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa6a00 x1876702068136448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.867502:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068136448/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.867509:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.867511:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.867515:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136448:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766189.867520:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766189.867538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.867549:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa5500 x1876702068136576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.867560:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068136576/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.867568:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.867571:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.867574:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068136576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766189.867867:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068136704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766189.867890:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766189.867893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766189.867895:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766189.867899:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3657 00000100:00080000:3.0:1789766189.867903:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766189.867908:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766189.868011:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766189.868026:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068136832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766189.868253:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068136704:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766189.868282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.868295:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa5180 x1876702068136704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.868307:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068136704/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.868315:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.868318:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.868320:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068136704:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766189.868326:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068136832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766189.868345:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766189.868356:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766189/real 1789766189] req@ffff936010aa4700 x1876702068136832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766189.868367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068136832/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766189.868375:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766189.868378:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766189.868381:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068136832:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766190.250924:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766191.275214:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766192.297950:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766193.324766:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766194.346537:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766194.987077:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987088:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766194.987092:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987101:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987112:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987120:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766194.987159:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068136960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987193:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987197:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766194.987199:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987203:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987206:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987209:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987219:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766194.987231:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068137088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987240:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987243:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766194.987245:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987248:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987251:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987255:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987260:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766194.987282:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068137216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987292:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987295:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766194.987297:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987301:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987312:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987318:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987323:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766194.987333:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068137344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987342:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987344:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766194.987347:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987350:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987353:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987356:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987360:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766194.987369:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068137472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987377:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987380:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766194.987382:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987386:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987389:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766194.987393:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068136960:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766194.987396:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987404:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766194.987414:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068137600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766194.987424:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766194.987427:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766194.987429:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766194.987432:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3662 00000100:00080000:2.0:1789766194.987437:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766194.987440:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766194.987444:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766194.987456:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068137728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766194.987618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766194.987651:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766194.987682:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766194.987705:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766194.987733:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068137600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766194.987753:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068137728:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766195.370109:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766196.394102:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766197.418610:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789766198.060129:0:2383:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:3.0:1789766198.060173:0:2383:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789766198.060189:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060234:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060243:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060251:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060262:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060271:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:3.0:1789766198.060278:0:2383:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:0.0:1789766198.060371:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061ddf80 x1876702068137600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068137600/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060414:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060422:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068137600:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766198.060430:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061dc700 x1876702068137728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060442:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068137728/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060449:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060455:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068137728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766198.060461:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061de300 x1876702068136960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060473:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068136960/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060480:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060483:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060486:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068136960:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766198.060492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061df100 x1876702068137088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068137088/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060512:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060516:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137088:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766198.060522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061ddc00 x1876702068137216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068137216/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060540:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060547:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766198.060553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061dd880 x1876702068137344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068137344/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060572:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060575:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060578:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137344:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766198.060585:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766194/real 1789766198] req@ffff9360061dc380 x1876702068137472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766198.060596:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068137472/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766198.060603:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766198.060606:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766198.060609:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137472:192.168.203.137@tcp:8:kworker.0 00010000:00010000:0.0:1789766198.442157:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766199.466251:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766200.106288:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766200.106302:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106310:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106321:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106327:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766200.106358:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068137856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106381:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766200.106386:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106389:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106393:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106396:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106405:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766200.106415:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068137984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106425:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106440:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766200.106442:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106446:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106450:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106454:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106458:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766200.106468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068138112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766200.106481:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106484:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106487:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106490:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106494:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766200.106503:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068138240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106512:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106515:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766200.106517:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106520:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106523:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106526:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106530:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766200.106540:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068138368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106555:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106559:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766200.106560:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106565:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106568:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106571:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106576:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766200.106585:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068138496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766200.106665:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766200.106668:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766200.106670:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766200.106673:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3667 00000100:00080000:3.0:1789766200.106676:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766200.106680:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766200.106685:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766200.106695:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068138624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766200.106853:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068137856:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766200.106911:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.106959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa4380 x1876702068137856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.106978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068137856/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.106989:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.106994:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.106997:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068137856:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766200.107004:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766200.107026:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa4e00 x1876702068137984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107050:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068137984/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107062:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107065:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107068:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068137984:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766200.107072:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766200.107090:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107100:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa7100 x1876702068138112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107111:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068138112/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107118:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107121:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107124:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766200.107130:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138240:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766200.107149:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107159:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa4700 x1876702068138240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107170:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068138240/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107177:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107180:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107183:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138240:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766200.107189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766200.107207:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107218:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa5180 x1876702068138368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107228:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068138368/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107236:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107239:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107242:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766200.107248:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068138496:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766200.107266:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107276:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa5500 x1876702068138496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107302:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068138496/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107309:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107312:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107315:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068138496:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766200.107321:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068138624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766200.107341:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766200.107352:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766200/real 1789766200] req@ffff936010aa6a00 x1876702068138624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766200.107363:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068138624/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766200.107370:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766200.107373:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766200.107375:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068138624:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766200.490196:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766201.514538:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766202.538279:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766203.562978:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766204.585976:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766205.225968:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.225978:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766205.225981:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.225991:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226001:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226008:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766205.226043:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068138752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766205.226068:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.226071:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766205.226073:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226076:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226080:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226084:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766205.226092:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766205.226105:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068138880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766205.226116:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789766205.226118:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068138752:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766205.226121:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766205.226125:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226130:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226134:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226137:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766205.226142:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766205.226153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068139008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766205.226163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.226166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766205.226168:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226171:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226175:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226179:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000800:00000100:0.0:1789766205.226184:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000008:00100000:3.0:1789766205.226184:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766205.226200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068139136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766205.226209:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.226212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766205.226214:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226218:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226221:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226224:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766205.226228:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766205.226237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa6680 x1876702068138752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766205.226240:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068139264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766205.226252:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.226255:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766205.226258:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226265:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226269:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766205.226276:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226277:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068138752/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789766205.226282:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789766205.226293:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226299:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766205.226300:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068139392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766205.226306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068138752:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766205.226318:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766205.226321:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766205.226322:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766205.226326:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3673 00000100:00080000:3.0:1789766205.226330:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789766205.226335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138880:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766205.226335:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766205.226343:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766205.226356:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068139520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766205.226366:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226380:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa7480 x1876702068138880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.226399:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068138880/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.226407:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226410:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.226413:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068138880:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766205.226419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766205.226437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa5f80 x1876702068139008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.226459:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068139008/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.226467:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226470:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.226474:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139008:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766205.226479:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139136:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766205.226499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa4a80 x1876702068139136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.226521:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068139136/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.226537:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226540:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.226543:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139136:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766205.226549:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766205.226568:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226578:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa5880 x1876702068139264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.226914:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068139264/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.226922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.226928:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766205.226934:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068139392:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766205.226955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.226967:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa4000 x1876702068139392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.226978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068139392/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.226986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.226990:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.226994:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068139392:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766205.227015:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068139520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766205.227034:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766205.227044:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766205/real 1789766205] req@ffff936010aa7800 x1876702068139520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766205.227055:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068139520/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766205.227063:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766205.227066:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766205.227069:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068139520:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766205.611703:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766206.634061:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766207.658247:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766208.683722:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766209.706982:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766210.346105:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346114:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766210.346118:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346127:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346137:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346143:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766210.346173:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068139648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766210.346212:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346215:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346219:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346222:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766210.346241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068139776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346250:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346253:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766210.346255:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346258:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346261:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346264:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346267:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766210.346587:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068139904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346596:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346600:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766210.346601:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346605:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346609:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346613:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346618:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766210.346628:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068140032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346635:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346637:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766210.346639:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346642:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346644:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346646:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346649:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766210.346656:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068140160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346663:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766210.346665:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766210.346667:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346670:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346673:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346676:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346680:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766210.346689:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068140288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766210.346698:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789766210.346699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068139648:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766210.346702:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766210.346705:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766210.346711:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3678 00000100:00080000:3.0:1789766210.346714:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766210.346717:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766210.346721:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766210.346731:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068140416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789766210.346760:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.346834:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa5c00 x1876702068139648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.346853:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068139648/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.346863:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.346867:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.346871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068139648:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766210.346878:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766210.346902:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.346913:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa7b80 x1876702068139776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.346925:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068139776/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.346933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.346936:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.346939:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139776:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766210.346945:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139904:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766210.346963:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.346973:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa6300 x1876702068139904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.346983:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068139904/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.346990:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.346992:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.346995:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068139904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766210.347000:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140032:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766210.347018:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.347029:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa7800 x1876702068140032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.347045:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068140032/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.347053:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.347056:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.347059:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140032:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766210.347064:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766210.347082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.347092:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa4000 x1876702068140160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.347101:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068140160/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.347108:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.347110:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.347113:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766210.347142:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068140288:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766210.347159:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.347169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa5880 x1876702068140288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.347180:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068140288/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.347192:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.347195:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.347198:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068140288:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766210.347203:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068140416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766210.347220:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766210.347231:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766210/real 1789766210] req@ffff936010aa4a80 x1876702068140416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766210.347241:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068140416/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766210.347247:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766210.347250:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766210.347253:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068140416:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766210.731256:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766211.754573:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766212.778113:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766213.802012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766214.826355:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766215.465965:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.465975:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766215.465979:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.465989:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466013:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466021:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766215.466056:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068140544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466086:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466089:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766215.466091:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466095:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466098:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466102:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466111:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766215.466123:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068140672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466132:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466136:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766215.466138:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466141:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466144:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466148:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466152:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766215.466165:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068140800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466174:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466178:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766215.466179:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466183:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466186:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466190:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466195:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766215.466205:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068140928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466214:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466217:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766215.466219:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466222:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466225:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466229:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466233:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766215.466242:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068141056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466250:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466253:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766215.466255:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466258:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466261:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466264:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466269:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766215.466279:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068141184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766215.466287:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766215.466290:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766215.466292:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766215.466295:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3683 00000100:00080000:1.0:1789766215.466298:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766215.466301:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766215.466306:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766215.466315:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068141312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766215.466415:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068140544:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766215.466473:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466530:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef7800 x1876702068140544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466548:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068140544/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466558:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466562:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466566:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068140544:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766215.466574:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140672:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766215.466597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef5500 x1876702068140672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466620:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068140672/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140672:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766215.466638:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766215.466657:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef5f80 x1876702068140800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466679:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068140800/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466686:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466690:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466693:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766215.466698:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140928:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766215.466716:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef6a00 x1876702068140928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068140928/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466743:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068140928:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766215.466762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766215.466780:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466824:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef6680 x1876702068141056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466837:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068141056/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466844:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466847:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466850:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766215.466927:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068141184:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766215.466951:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.466962:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef7480 x1876702068141184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.466978:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068141184/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.466986:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.466988:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.466991:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068141184:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766215.466997:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068141312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766215.467014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766215.467024:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766215/real 1789766215] req@ffff936010ef7b80 x1876702068141312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766215.467035:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068141312/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766215.467041:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766215.467044:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766215.467047:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068141312:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766215.849981:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766216.874281:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766217.898041:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766218.923392:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766219.946034:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766220.585916:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.585926:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766220.585930:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.585939:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.585949:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.585956:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766220.585992:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068141440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766220.586057:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068141440:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766220.586112:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766220.586139:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.586143:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766220.586146:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.586150:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.586154:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.586157:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.586166:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766220.586178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068141568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766220.588165:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef4e00 x1876702068141440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.588193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068141440/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.588205:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.588210:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.588214:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068141440:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766220.588243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141568:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766220.588295:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766220.588312:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef7100 x1876702068141568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.588324:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068141568/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.588333:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.588336:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.588339:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141568:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766220.588533:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.588538:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766220.588541:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.588547:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.588712:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.588720:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.588730:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766220.588981:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068141696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766220.589034:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766220.589063:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766220.589089:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.589093:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766220.589095:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.589099:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.589104:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.589109:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.589114:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766220.589128:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068141824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766220.589209:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef4700 x1876702068141696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.589223:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068141696/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.589231:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.589235:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.589238:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141696:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766220.589301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.589304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766220.589305:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.589310:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.589314:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.589318:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.589323:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766220.589335:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068141952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766220.589344:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.589346:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766220.589348:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.589352:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.589355:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.589358:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.589362:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766220.589372:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068142080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766220.589380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766220.589383:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766220.589384:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766220.589388:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3688 00000100:00080000:1.0:1789766220.589391:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766220.589394:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766220.589399:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766220.589408:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068142208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766220.589474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141824:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766220.589509:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766220.589522:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef5180 x1876702068141824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.589534:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068141824/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.589541:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.589544:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.589548:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141824:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766220.589644:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766220.589668:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766220.589679:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef5c00 x1876702068141952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.589690:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068141952/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.589698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.589701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.589976:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068141952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766220.589986:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142080:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766220.590011:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766220.590023:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef6d80 x1876702068142080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.590035:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068142080/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.590043:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.590046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.590049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142080:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766220.590054:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766220.590072:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766220.590083:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766220/real 1789766220] req@ffff936010ef6300 x1876702068142208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766220.590094:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068142208/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766220.590101:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766220.590104:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766220.590107:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142208:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766220.970020:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766221.997145:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766223.018012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766224.042692:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766225.066002:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766225.705996:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706006:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766225.706009:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706019:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706030:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706037:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766225.706076:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068142336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706103:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706107:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766225.706108:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706113:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706116:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706120:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706129:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766225.706141:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068142464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706149:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706152:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766225.706154:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706157:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706160:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706163:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706167:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766225.706176:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068142592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706184:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706187:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766225.706189:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706192:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706194:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706197:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766225.706219:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068142720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706226:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706229:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766225.706231:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706233:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706236:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706239:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766225.706249:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068142848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706255:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706258:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766225.706259:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706266:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706268:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706271:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706275:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766225.706283:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068142976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766225.706290:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766225.706292:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766225.706294:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766225.706296:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3693 00000100:00080000:3.0:1789766225.706299:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766225.706302:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766225.706305:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766225.706312:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068143104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766225.706403:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068142336:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766225.706479:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706523:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa5f80 x1876702068142336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706554:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068142336/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706564:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706568:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706571:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068142336:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766225.706578:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142464:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766225.706600:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706612:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa7480 x1876702068142464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706624:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068142464/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706631:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706634:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706637:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142464:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766225.706643:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766225.706670:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706681:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa6680 x1876702068142592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706691:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068142592/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706698:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706701:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706704:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766225.706710:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142720:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766225.706729:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706741:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa6a00 x1876702068142720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068142720/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706772:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706778:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142720:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766225.706783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766225.706833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa5500 x1876702068142848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706854:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068142848/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706861:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706864:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706867:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068142848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766225.706873:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142976:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766225.706891:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706904:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa5180 x1876702068142976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706916:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068142976/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706923:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706926:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706929:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068142976:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766225.706934:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068143104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766225.706953:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766225.706963:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766225/real 1789766225] req@ffff936010aa4700 x1876702068143104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766225.706973:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068143104/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766225.706981:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766225.706983:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766225.706986:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068143104:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766226.090232:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766227.114087:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766228.138198:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766229.162005:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766230.186397:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766230.825940:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.825950:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766230.825955:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.825965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.825977:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.825985:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766230.826025:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068143232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826060:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766230.826066:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826069:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826073:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826077:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826086:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766230.826099:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068143360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826108:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826112:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766230.826113:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826118:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826122:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826126:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826131:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766230.826141:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068143488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826150:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826152:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766230.826154:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826157:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826160:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826163:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826167:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766230.826178:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068143616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766230.826192:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826196:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826199:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826203:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826207:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766230.826216:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068143744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826225:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826228:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766230.826229:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826232:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826235:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826239:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826243:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766230.826252:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068143872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766230.826260:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766230.826263:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766230.826265:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766230.826274:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3698 00000100:00080000:3.0:1789766230.826277:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766230.826281:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766230.826285:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766230.826295:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068144000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766230.826588:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068143232:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766230.826645:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.826841:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa7100 x1876702068143232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.826859:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068143232/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.826871:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.826875:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.826879:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068143232:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766230.826886:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143360:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766230.826909:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.826922:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa4e00 x1876702068143360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.826932:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068143360/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.826940:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.826943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.826946:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143360:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766230.826952:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143488:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766230.826970:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.826981:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa4380 x1876702068143488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.826991:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068143488/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.826998:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.827001:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.827004:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766230.827010:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143616:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766230.827028:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.827038:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa4700 x1876702068143616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.827048:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068143616/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.827055:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.827058:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.827060:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143616:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766230.827066:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766230.827093:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.827104:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa5180 x1876702068143744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.827115:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068143744/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.827123:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.827126:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.827130:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068143744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766230.827135:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068143872:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766230.827153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.827164:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa5500 x1876702068143872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.827176:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068143872/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.827183:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.827186:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.827190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068143872:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766230.827196:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766230.827214:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766230.827224:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766230/real 1789766230] req@ffff936010aa6a00 x1876702068144000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766230.827235:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068144000/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766230.827241:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766230.827244:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766230.827247:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144000:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766231.210388:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766232.234669:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766233.258729:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766234.282460:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766235.306436:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766235.946303:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.946318:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766235.946321:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.946331:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.946353:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.946362:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766235.946406:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068144128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766235.946474:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068144128:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766235.946530:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766235.946560:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.946564:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766235.946566:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.946570:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.946574:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.946578:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.946587:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766235.946600:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068144256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766235.946654:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef4000 x1876702068144128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.946672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068144128/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.946682:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.946686:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.946689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068144128:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766235.946709:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144256:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766235.946738:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766235.946750:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef4380 x1876702068144256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.946763:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068144256/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.946770:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.946774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.946777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144256:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766235.950397:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.950407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766235.950410:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.950420:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.950429:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.950437:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.950510:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766235.950543:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068144384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766235.950605:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144384:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766235.950660:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766235.950690:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.950694:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766235.950697:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.950701:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.950705:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.950709:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.950714:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766235.950726:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068144512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766235.950772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef5880 x1876702068144384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766235.950890:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.950893:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766235.950895:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.950897:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.950900:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.950903:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.950907:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766235.950916:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068144640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766235.950922:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.950924:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766235.950925:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.950927:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.950929:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.950931:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.950933:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766235.950937:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068144768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766235.950942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766235.950944:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766235.950945:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766235.950947:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3703 00000100:00080000:1.0:1789766235.950948:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766235.950950:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766235.950952:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766235.950956:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068144896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766235.951082:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068144384/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.951093:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.951097:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.951100:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766235.951131:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144512:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766235.951154:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766235.951166:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef4a80 x1876702068144512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.951177:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068144512/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.951185:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.951188:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.951191:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144512:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766235.951196:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144640:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766235.951219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766235.951230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef4380 x1876702068144640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.951254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068144640/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.951261:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.951263:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.951266:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068144640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766235.951272:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144768:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766235.951297:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766235.951308:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef4000 x1876702068144768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.951319:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068144768/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.951327:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.951329:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.951332:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144768:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766235.951338:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144896:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766235.951356:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766235.951366:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766235/real 1789766235] req@ffff936010ef6300 x1876702068144896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766235.951392:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068144896/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766235.951399:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766235.951402:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766235.951405:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068144896:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766236.332639:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766237.357392:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766238.378610:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766239.403418:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766240.426086:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766241.065975:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.065985:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766241.065990:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.065999:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.066010:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.066018:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766241.066054:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068145024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766241.066122:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068145024:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766241.066178:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766241.066210:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.066215:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766241.066216:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.066220:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.066224:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.066228:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.066237:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766241.066248:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068145152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766241.067448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa6680 x1876702068145024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.067469:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068145024/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.067480:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.067484:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.067488:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068145024:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766241.067513:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145152:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766241.067549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.067564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa7480 x1876702068145152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.067576:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068145152/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.067585:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.067588:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.067591:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145152:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766241.071501:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.071512:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766241.071515:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.071525:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.071535:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.071542:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.071554:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766241.071586:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068145280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766241.071607:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.071610:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766241.071612:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.071616:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.071619:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.071623:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.071627:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766241.071636:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068145408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766241.071645:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.071649:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766241.071650:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.071654:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.071657:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.071660:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.071665:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766241.071673:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068145536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766241.071682:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.071685:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766241.071687:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.071690:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.071693:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.071697:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.071701:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766241.071712:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068145664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766241.071733:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766241.071737:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766241.071738:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766241.071742:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3708 00000100:00080000:3.0:1789766241.071745:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766241.071748:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766241.071753:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766241.071762:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068145792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766241.072052:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766241.072107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.072132:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa5f80 x1876702068145280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.072147:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068145280/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.072157:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.072161:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.072164:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145280:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766241.072171:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145408:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766241.072190:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.072201:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa4a80 x1876702068145408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.072212:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068145408/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.072219:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.072222:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.072226:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145408:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766241.072231:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766241.072250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.072260:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa5880 x1876702068145536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.072271:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068145536/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.072278:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.072281:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.072284:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068145536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766241.072290:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068145664:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766241.072308:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.072318:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa4000 x1876702068145664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.072329:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068145664/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.072352:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.072356:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.072359:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068145664:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766241.072364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068145792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766241.072385:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766241.072396:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766241/real 1789766241] req@ffff936010aa7800 x1876702068145792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766241.072408:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068145792/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766241.072415:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766241.072418:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766241.072421:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068145792:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766241.450373:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766242.474626:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766243.498217:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766244.522012:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766245.546506:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766246.185973:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.185983:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766246.185987:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.185996:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186006:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186013:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766246.186048:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068145920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186081:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186085:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766246.186086:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186090:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186094:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186097:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186105:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766246.186116:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068146048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186125:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186128:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766246.186129:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186133:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186136:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186139:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186143:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766246.186152:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068146176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186160:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186163:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766246.186165:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186169:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186172:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186175:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186178:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766246.186187:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068146304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186195:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186198:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766246.186199:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186202:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186205:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186208:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186211:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766246.186220:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068146432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186228:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186231:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766246.186233:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186236:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186239:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186242:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186246:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766246.186254:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068146560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766246.186262:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766246.186265:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766246.186267:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766246.186270:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3714 00000100:00080000:0.0:1789766246.186273:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766246.186275:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766246.186279:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766246.186288:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068146688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766246.186381:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068145920:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766246.186438:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186474:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020dac380 x1876702068145920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186490:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068145920/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186499:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186503:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186506:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068145920:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766246.186513:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146048:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766246.186534:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186546:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020dad880 x1876702068146048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186557:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068146048/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186564:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186567:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186570:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146048:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766246.186577:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146176:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766246.186596:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186607:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020dac000 x1876702068146176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186617:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068146176/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186624:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186627:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186630:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766246.186635:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146304:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766246.186656:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186666:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020dae680 x1876702068146304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186676:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068146304/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186687:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186690:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146304:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766246.186695:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766246.186713:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186724:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020daed80 x1876702068146432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186735:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068146432/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186746:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766246.186752:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068146560:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766246.186771:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020daf480 x1876702068146560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186792:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068146560/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186828:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186832:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186835:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068146560:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766246.186841:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068146688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766246.186861:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766246.186872:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766246/real 1789766246] req@ffff936020dafb80 x1876702068146688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766246.186883:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068146688/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766246.186896:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766246.186899:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766246.186902:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068146688:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766246.571334:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766247.595504:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766248.618068:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766249.642006:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766250.668116:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766251.305936:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.305946:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766251.305950:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.305959:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.305970:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.305977:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766251.306013:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068146816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306053:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306057:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766251.306058:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306063:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306067:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306070:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306079:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766251.306091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068146944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306101:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306104:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766251.306105:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306109:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306112:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306116:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306121:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766251.306132:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068147072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306140:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306143:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766251.306144:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306147:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306150:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306153:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306157:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766251.306166:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068147200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306173:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306176:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766251.306178:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306181:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306184:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306186:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306190:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766251.306199:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068147328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306206:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766251.306211:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306213:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306216:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306220:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306224:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766251.306232:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068147456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766251.306240:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766251.306254:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766251.306265:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766251.306269:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3719 00000100:00080000:2.0:1789766251.306272:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766251.306275:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766251.306279:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766251.306293:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068147584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766251.306380:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068146816:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766251.306436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306477:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061df480 x1876702068146816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306494:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068146816/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306503:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306507:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306511:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068146816:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766251.306517:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146944:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766251.306538:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306550:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061dd500 x1876702068146944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306561:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068146944/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306569:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306573:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306575:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068146944:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766251.306581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766251.306599:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306609:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061dc000 x1876702068147072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306619:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068147072/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306626:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306628:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306631:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147072:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766251.306636:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147200:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766251.306653:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061dce00 x1876702068147200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306677:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068147200/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306684:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306687:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306689:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147200:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766251.306694:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766251.306711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306720:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061dd180 x1876702068147328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306730:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068147328/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306741:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147328:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766251.306746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068147456:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766251.306761:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306769:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061df800 x1876702068147456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306779:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068147456/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306785:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306788:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306790:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068147456:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766251.306846:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068147584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766251.306876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766251.306886:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766251/real 1789766251] req@ffff9360061dca80 x1876702068147584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766251.306901:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068147584/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766251.306908:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766251.306910:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766251.306913:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068147584:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766251.690030:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766252.714042:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766253.739621:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766254.762174:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766255.786687:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766256.425913:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.425924:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766256.425927:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.425936:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.425946:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.425953:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766256.425988:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068147712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426018:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426022:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766256.426023:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426027:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426030:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426034:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426041:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766256.426053:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068147840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426062:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426065:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766256.426066:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426069:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426072:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426075:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426079:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766256.426091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068147968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426099:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426102:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766256.426104:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426107:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426110:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426114:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426118:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766256.426127:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068148096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426135:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426138:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766256.426140:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426143:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426146:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426149:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426153:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766256.426163:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068148224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426171:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426174:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766256.426176:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426179:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426182:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426185:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426189:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766256.426198:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068148352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766256.426208:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766256.426210:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766256.426211:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766256.426213:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3724 00000100:00080000:2.0:1789766256.426215:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766256.426358:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766256.426364:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766256.426378:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068148480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766256.426382:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068147712:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766256.426454:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.426495:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061de680 x1876702068147712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.426511:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068147712/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.426521:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.426525:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.426528:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068147712:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766256.426612:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147840:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766256.426711:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.426725:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061dfb80 x1876702068147840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.426736:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068147840/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.426743:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.426746:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.426749:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147840:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766256.426754:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147968:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766256.426774:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.426786:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061dca80 x1876702068147968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.426904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068147968/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.426913:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.426916:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.426919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068147968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766256.426926:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148096:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766256.426949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.426960:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061df800 x1876702068148096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.426971:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068148096/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.426979:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.426982:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.426985:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148096:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766256.426991:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766256.427014:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.427025:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061dd180 x1876702068148224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.427036:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068148224/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.427043:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.427046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.427049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766256.427054:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068148352:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766256.427073:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.427084:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061dce00 x1876702068148352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.427096:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068148352/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.427103:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.427106:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.427109:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068148352:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766256.427138:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068148480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766256.427157:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766256.427169:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766256/real 1789766256] req@ffff9360061dc000 x1876702068148480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766256.427180:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068148480/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766256.427188:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766256.427191:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766256.427193:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068148480:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766256.810283:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766257.834042:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766258.858251:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766259.882595:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766260.907261:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766261.547013:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547023:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766261.547027:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.547037:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547048:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547056:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766261.547091:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068148608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766261.547188:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547192:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766261.547195:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.547199:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547203:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547207:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.547217:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766261.547232:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068148736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766261.547238:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068148608:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766261.547248:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547252:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766261.547254:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.547259:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547263:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547268:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.547273:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766261.547285:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068148864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766261.547294:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547298:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766261.547300:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.547303:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547307:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547311:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.547316:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766261.547329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068148992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766261.547339:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547342:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766261.547344:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766261.547347:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148736:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766261.547351:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547358:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547362:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.547367:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766261.547384:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766261.547406:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148992:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766261.547524:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068149120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766261.547593:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149120:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789766261.547642:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.547646:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766261.547649:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.547653:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.547658:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.547662:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.547668:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766261.547682:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068149248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766261.553866:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068149248:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766261.555012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766261.555021:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766261.555024:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766261.555032:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3729 00000100:00080000:3.0:1789766261.555041:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766261.555048:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766261.555059:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766261.555159:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068149376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766261.555223:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068149376:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766261.930084:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766262.958004:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766263.978003:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789766264.618553:0:2384:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789766264.618595:0:2384:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789766264.618610:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789766264.622056:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa6300 x1876702068148608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766286 ref 2 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622088:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068148608/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766286 ref 2 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622179:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622185:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622190:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068148608:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789766264.622439:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766264.622470:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766264.622479:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766264.622485:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766264.622493:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766264.622499:0:2384:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:1.0:1789766264.622588:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa4a80 x1876702068149376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622604:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068149376/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622616:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068149376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766264.622627:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa5880 x1876702068149248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068149248/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622649:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622652:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068149248:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766264.622657:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa4000 x1876702068149120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622668:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068149120/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622676:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622680:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622683:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766264.622689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa7b80 x1876702068148736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068148736/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622706:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622709:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622713:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148736:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766264.622719:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa5c00 x1876702068148864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622729:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068148864/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622736:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622739:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622742:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766264.622748:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766261/real 1789766264] req@ffff936010aa7800 x1876702068148992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766264.622759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068148992/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766264.622767:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766264.622770:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766264.622773:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068148992:192.168.203.137@tcp:8:kworker.0 00010000:00010000:1.0:1789766265.001959:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766266.026006:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766266.667199:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667211:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766266.667215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667226:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667236:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667243:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766266.667277:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068149504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667311:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667315:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766266.667317:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667321:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667324:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667327:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667336:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766266.667346:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068149632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667355:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667359:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766266.667360:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667364:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667367:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667370:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667374:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766266.667383:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068149760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667391:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667394:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766266.667396:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667399:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667402:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667405:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667410:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766266.667419:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068149888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667426:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667429:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766266.667431:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667434:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667437:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667440:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667444:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766266.667453:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068150016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667460:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667463:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766266.667465:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667468:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667470:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667473:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667477:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766266.667485:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068150144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766266.667492:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766266.667494:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766266.667496:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766266.667499:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3734 00000100:00080000:2.0:1789766266.667502:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766266.667505:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766266.667509:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766266.667518:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068150272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766266.668187:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068149504:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766266.668250:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668291:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061dd500 x1876702068149504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668309:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068149504/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668320:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668324:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668328:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068149504:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766266.668335:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149632:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766266.668357:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668369:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061df480 x1876702068149632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668380:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068149632/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668388:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668392:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668395:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149632:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766266.668401:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766266.668419:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668429:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061dc380 x1876702068149760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668441:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068149760/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668449:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668452:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668456:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766266.668462:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149888:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766266.668482:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668492:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061dd880 x1876702068149888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668504:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068149888/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068149888:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766266.668524:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766266.668542:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668553:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061ddc00 x1876702068150016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668565:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068150016/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668573:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668576:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668579:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766266.668585:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068150144:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766266.668604:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668614:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061df100 x1876702068150144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668626:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068150144/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668634:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668637:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668640:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068150144:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766266.668646:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068150272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766266.668666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766266.668676:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766266/real 1789766266] req@ffff9360061de300 x1876702068150272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766266.668688:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068150272/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766266.668696:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766266.668699:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766266.668702:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068150272:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766267.049991:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766268.073993:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766269.103702:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766270.122234:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766271.146077:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766271.785976:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.785987:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766271.785992:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766271.786002:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786013:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786020:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766271.786057:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068150400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766271.786112:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766271.786119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766271.786123:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786127:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786131:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786140:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766271.786153:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068150528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766271.786158:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068150400:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766271.786167:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786171:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766271.786173:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766271.786177:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786181:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786184:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786189:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766271.786201:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068150656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766271.786211:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786214:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766271.786215:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789766271.786221:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766271.786222:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786229:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786234:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786239:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766271.786250:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068150784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766271.786258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766271.786263:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766271.786268:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa5f80 x1876702068150400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766271.786269:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786275:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786282:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786288:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766271.786295:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068150400/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766271.786301:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068150912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766271.786316:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766271.786317:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786323:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766271.786325:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:3.0:1789766271.786326:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766271.786333:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068150400:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766271.786334:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3739 00000100:00080000:3.0:1789766271.786340:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786346:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786351:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766271.786362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068151040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766271.786364:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150528:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766271.786380:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766271.786384:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766271.786386:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766271.786391:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3739 00000800:00000100:1.0:1789766271.786399:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766271.786403:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766271.786410:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766271.786416:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766271.786418:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa7480 x1876702068150528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786431:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068150528/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786439:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789766271.786441:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068151168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766271.786446:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786454:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150528:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766271.786464:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766271.786486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766271.786500:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa6680 x1876702068150656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786511:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068150656/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786519:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766271.786522:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786525:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766271.786531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150784:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766271.786549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766271.786574:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa6a00 x1876702068150784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786590:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068150784/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786598:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766271.786600:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786603:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150784:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766271.786609:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766271.786628:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766271.786639:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa5500 x1876702068150912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786650:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068150912/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786657:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766271.786660:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786663:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068150912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766271.786679:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151040:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766271.786705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766271.786792:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa5180 x1876702068151040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786836:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068151040/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786843:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766271.786846:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786849:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151040:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766271.786855:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766271.786876:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766271.786887:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766271/real 1789766271] req@ffff936010aa4700 x1876702068151168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766271.786904:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068151168/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766271.786912:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766271.786916:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766271.786919:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151168:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766272.175252:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766273.194001:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766274.218018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766275.241992:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766276.265985:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766276.905917:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.905928:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766276.905931:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.905941:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.905950:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.905958:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766276.905991:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068151296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906021:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906025:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766276.906027:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906031:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906034:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906037:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906045:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766276.906058:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068151424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906078:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906081:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766276.906083:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906087:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906091:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906094:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906098:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766276.906110:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068151552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906119:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906122:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766276.906123:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906126:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906129:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906132:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906136:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766276.906146:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068151680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906155:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906158:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766276.906160:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906164:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906166:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906170:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906174:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766276.906182:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068151808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906190:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906193:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766276.906195:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906198:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906204:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906208:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766276.906216:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068151936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766276.906229:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766276.906232:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766276.906234:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766276.906237:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3744 00000100:00080000:3.0:1789766276.906240:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766276.906244:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766276.906248:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766276.906257:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068152064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766276.906349:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068151296:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766276.906403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906437:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa4380 x1876702068151296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906453:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068151296/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906463:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068151296:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766276.906478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151424:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766276.906499:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906510:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa4e00 x1876702068151424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906535:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068151424/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906549:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906552:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906555:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151424:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766276.906561:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766276.906591:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906602:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa7100 x1876702068151552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068151552/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906622:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906625:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906628:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766276.906634:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151680:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766276.906651:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906661:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa4700 x1876702068151680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906672:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068151680/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906679:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906682:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906686:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151680:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766276.906692:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766276.906707:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906717:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa5180 x1876702068151808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068151808/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906734:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906737:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068151808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766276.906746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151936:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766276.906763:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906772:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa5500 x1876702068151936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906783:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068151936/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906791:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906794:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906797:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068151936:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766276.906831:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766276.906849:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766276.906860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766276/real 1789766276] req@ffff936010aa6a00 x1876702068152064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766276.906871:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068152064/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766276.906880:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766276.906883:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766276.906886:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152064:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766277.290171:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766278.314272:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766279.338103:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766280.362433:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766281.386309:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766282.026084:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026094:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766282.026098:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026108:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026118:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026126:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766282.026163:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068152192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026192:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026195:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766282.026197:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026201:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026204:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026207:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026216:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766282.026227:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068152320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026237:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766282.026239:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026242:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026244:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026247:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026251:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766282.026259:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068152448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026268:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766282.026270:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026273:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026276:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026280:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026284:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766282.026293:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068152576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766282.026305:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026308:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026311:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026315:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026319:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766282.026327:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068152704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026335:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026338:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766282.026339:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026342:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026345:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026348:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026351:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766282.026359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068152832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766282.026366:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766282.026368:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766282.026369:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766282.026372:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3749 00000100:00080000:1.0:1789766282.026375:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766282.026378:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766282.026382:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766282.026390:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068152960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789766282.026466:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068152192:192.168.203.137@tcp:250:kworker.0 00000800:00000100:0.0:1789766282.026522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026559:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef6d80 x1876702068152192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026582:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068152192/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026592:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026596:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068152192:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766282.026606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152320:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766282.026626:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026636:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef5c00 x1876702068152320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026646:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068152320/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026653:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026656:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026659:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152320:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766282.026664:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766282.026681:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef5180 x1876702068152448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026701:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068152448/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026715:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026718:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026721:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152448:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766282.026726:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152576:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766282.026743:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026753:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef4700 x1876702068152576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026764:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068152576/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026774:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026777:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152576:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766282.026783:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:0.0:1789766282.026830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026841:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef7100 x1876702068152704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026851:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068152704/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026858:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026861:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026864:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068152704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:0.0:1789766282.026870:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152832:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766282.026889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026900:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef4e00 x1876702068152832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068152832/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026919:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026922:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026925:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152832:192.168.203.137@tcp:38:kworker.0 00000100:00100000:0.0:1789766282.026930:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:0.0:1789766282.026949:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:0.0:1789766282.026959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766282/real 1789766282] req@ffff936010ef7b80 x1876702068152960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766282.026970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068152960/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766282.026977:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789766282.026980:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:0.0:1789766282.026983:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068152960:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766282.409980:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766283.433976:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766284.458327:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766285.481997:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766286.506046:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766287.145953:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.145964:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766287.145968:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.145977:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.145988:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146006:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766287.146043:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf100 x1876702068153088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766287.146072:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.146076:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766287.146078:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.146081:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.146084:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146088:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766287.146096:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766287.146106:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac700 x1876702068153216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766287.146114:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.146117:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766287.146119:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.146122:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.146125:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146128:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766287.146131:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766287.146139:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad500 x1876702068153344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766287.146147:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.146149:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766287.146151:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.146154:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.146156:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146159:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766287.146163:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766287.146172:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068153472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766287.146180:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.146183:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766287.146185:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.146187:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.146190:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146192:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766287.146196:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766287.146203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068153600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766287.146210:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.146213:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766287.146215:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.146218:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00080000:0.0:1789766287.146221:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.146225:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146379:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068153088:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766287.146446:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146484:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020daf100 x1876702068153088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.146500:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf100 x1876702068153088/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.146510:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.146514:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.146518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068153088:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766287.146525:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153216:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766287.146549:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146563:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020dac700 x1876702068153216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.146575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac700 x1876702068153216/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.146583:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.146586:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.146590:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153216:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766287.146596:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766287.146615:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146626:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020dad500 x1876702068153344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.146637:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad500 x1876702068153344/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.146647:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.146650:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.146653:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766287.146658:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153472:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766287.146678:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020dafb80 x1876702068153472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.146699:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068153472/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.146877:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.146881:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.146885:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153472:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766287.146891:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766287.146914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.146926:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020daf480 x1876702068153600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.146944:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068153600/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.146950:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.146953:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.146955:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068153600:192.168.203.137@tcp:8:kworker.0 00000008:00100000:0.0:1789766287.147922:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766287.148020:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068153728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766287.148093:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068153728:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766287.148139:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:0.0:1789766287.148366:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766287.148371:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766287.148374:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766287.148381:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3754 00000100:00100000:1.0:1789766287.148383:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020daed80 x1876702068153728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789766287.148387:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766287.148397:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766287.148403:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766287.148416:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068153728/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789766287.148421:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068153856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766287.148435:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.148442:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.148447:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068153728:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766287.148469:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068153856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766287.148493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766287.148507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766287/real 1789766287] req@ffff936020dae680 x1876702068153856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766287.148519:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068153856/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766287.148532:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766287.148535:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766287.148538:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068153856:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766287.530023:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766288.554221:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766289.578148:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766290.602075:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766291.626245:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766292.265920:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.265928:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766292.265931:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.265940:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.265950:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.265968:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766292.266003:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068153984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266041:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266045:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766292.266047:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266051:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266054:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266057:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266065:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766292.266077:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068154112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266084:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266087:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766292.266089:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266093:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266095:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266098:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266101:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766292.266109:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068154240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266117:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266119:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766292.266121:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266124:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266127:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266131:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266135:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766292.266143:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068154368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266151:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266154:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766292.266155:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266158:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266161:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266164:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266167:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766292.266175:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068154496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266183:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266186:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766292.266187:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266191:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266193:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266196:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266200:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766292.266208:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068154624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766292.266222:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766292.266225:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766292.266226:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766292.266229:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3760 00000100:00080000:0.0:1789766292.266232:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766292.266235:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766292.266239:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766292.266261:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068154752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789766292.266348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068153984:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789766292.266403:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266439:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dac000 x1876702068153984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266454:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068153984/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266464:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266468:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266472:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068153984:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789766292.266478:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154112:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766292.266497:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dad880 x1876702068154112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068154112/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266524:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266526:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266529:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154112:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766292.266534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766292.266548:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266556:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dac380 x1876702068154240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266566:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068154240/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266572:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266574:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266577:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766292.266581:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154368:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766292.266597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266605:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dad180 x1876702068154368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266614:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068154368/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154368:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766292.266637:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766292.266654:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266664:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dae300 x1876702068154496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266674:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068154496/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266681:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266685:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266688:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068154496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766292.266693:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068154624:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766292.266718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020dace00 x1876702068154624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266741:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068154624/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266755:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068154624:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766292.266761:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068154752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766292.266779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766292.266789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766292/real 1789766292] req@ffff936020daea00 x1876702068154752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766292.266825:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068154752/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766292.266833:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766292.266836:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766292.266838:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068154752:192.168.203.137@tcp:38:kworker.0 00010000:00010000:1.0:1789766292.650022:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766293.674128:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766294.698142:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766295.722750:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766296.746204:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766297.385971:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.385982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766297.385986:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.385996:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.386007:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.386014:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766297.386298:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068154880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766297.386375:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068154880:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766297.386439:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789766297.386468:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.386472:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766297.386474:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.387247:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.387253:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.387259:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.387269:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766297.387286:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068155008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766297.387697:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061dc700 x1876702068154880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.387716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068154880/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.387725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.387729:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.387733:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068154880:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766297.387756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155008:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766297.387811:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.387826:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061ddf80 x1876702068155008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.387838:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068155008/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.387851:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.387854:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.387856:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155008:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766297.388368:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.388373:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766297.388375:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.388380:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.388385:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.388390:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.388396:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766297.388415:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068155136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766297.388425:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.388428:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766297.388430:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.388433:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.388436:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.388440:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.388445:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766297.388453:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068155264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766297.388461:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.388464:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766297.388466:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.388468:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.388472:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.388475:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.388478:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766297.388487:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068155392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766297.388494:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.388496:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766297.388497:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.388500:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.388502:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.388505:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.388508:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766297.388515:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068155520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766297.388522:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766297.388524:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766297.388525:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766297.388528:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3765 00000100:00080000:2.0:1789766297.388531:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766297.388534:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766297.388538:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766297.388545:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068155648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766297.390732:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766297.390830:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.390860:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061ddf80 x1876702068155136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.390878:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068155136/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.390889:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.390894:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.390897:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766297.390905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155264:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766297.390939:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.390951:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061dc700 x1876702068155264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.390963:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068155264/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.390971:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.390974:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.390978:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155264:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766297.390985:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766297.391003:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.391014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061de300 x1876702068155392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.391330:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068155392/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.391338:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.391341:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.391344:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766297.391351:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068155520:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766297.391375:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.391387:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061df100 x1876702068155520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.391397:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068155520/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.391405:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.391408:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.391411:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068155520:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766297.391418:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068155648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766297.391436:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766297.391447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766297/real 1789766297] req@ffff9360061ddc00 x1876702068155648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766297.391457:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068155648/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766297.391464:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766297.391467:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766297.391470:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068155648:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766297.770004:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789766298.794111:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766299.818029:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766300.843225:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766301.866053:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766302.505942:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.505952:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766302.505955:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766302.505965:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.505975:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.505982:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766302.506018:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7480 x1876702068155776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766302.506049:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.506052:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766302.506054:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766302.506058:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506062:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.506066:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506074:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766302.506086:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6680 x1876702068155904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766302.506097:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.506101:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766302.506103:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766302.506107:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506110:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.506114:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506120:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766302.506120:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068155776:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766302.506133:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6a00 x1876702068156032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766302.506144:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.506147:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766302.506149:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766302.506153:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506156:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.506160:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506164:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766302.506175:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5f80 x1876702068156160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766302.506183:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.506189:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:2.0:1789766302.506189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766302.506193:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766302.506198:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506202:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.506205:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506209:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766302.506220:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068156288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766302.506228:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766302.506231:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766302.506233:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789766302.506237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef7480 x1876702068155776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766302.506240:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506245:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766302.506253:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506259:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766302.506266:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7480 x1876702068155776/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766302.506271:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068156416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766302.506286:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766302.506289:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766302.506294:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789766302.506295:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766302.506300:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789766302.506303:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068155776:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789766302.506307:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3770 00000100:00080000:1.0:1789766302.506312:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789766302.506316:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155904:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766302.506318:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766302.506327:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766302.506340:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068156544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789766302.506351:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506367:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef6680 x1876702068155904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506380:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6680 x1876702068155904/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506388:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506391:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506394:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068155904:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766302.506415:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156032:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766302.506435:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef6a00 x1876702068156032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506456:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6a00 x1876702068156032/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506462:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506465:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506468:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766302.506473:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156160:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766302.506493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506503:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef5f80 x1876702068156160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506513:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5f80 x1876702068156160/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506520:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506523:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506526:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156160:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766302.506531:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156288:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766302.506550:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506564:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef5500 x1876702068156288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506580:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5500 x1876702068156288/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506587:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506590:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766302.506598:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068156416:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766302.506617:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506627:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef7800 x1876702068156416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506638:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7800 x1876702068156416/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506645:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506648:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506651:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068156416:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766302.506657:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068156544:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766302.506688:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766302.506700:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766302/real 1789766302] req@ffff936010ef7b80 x1876702068156544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766302.506711:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7b80 x1876702068156544/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766302.506724:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766302.506727:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766302.506730:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068156544:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766302.891636:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766303.915055:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766304.938018:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766305.962556:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766306.987139:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766307.625987:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.625997:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766307.626002:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626012:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626023:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626031:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766307.626067:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068156672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626100:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626104:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766307.626106:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626109:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626113:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626117:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626125:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766307.626138:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068156800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626148:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626151:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766307.626152:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626156:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626160:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626163:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626168:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766307.626178:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068156928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626187:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626190:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766307.626192:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626195:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626199:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626202:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626206:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766307.626216:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068157056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626224:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626227:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766307.626228:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626231:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626235:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626238:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626242:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766307.626252:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068157184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626260:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626263:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766307.626265:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626268:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626271:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626275:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626279:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766307.626290:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068157312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766307.626299:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766307.626302:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766307.626304:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766307.626307:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3775 00000100:00080000:3.0:1789766307.626310:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766307.626313:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766307.626318:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766307.626328:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068157440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789766307.626417:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068156672:192.168.203.137@tcp:250:kworker.0 00000800:00000100:2.0:1789766307.626476:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626515:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa6680 x1876702068156672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626532:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068156672/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626542:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626546:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626550:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068156672:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789766307.626556:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156800:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766307.626580:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626593:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa7480 x1876702068156800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626605:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068156800/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626613:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626617:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626620:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156800:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766307.626626:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156928:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766307.626647:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626659:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa5f80 x1876702068156928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626670:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068156928/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626677:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626681:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626684:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068156928:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766307.626689:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157056:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766307.626710:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626722:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa7800 x1876702068157056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626733:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068157056/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626741:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626744:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626747:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157056:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766307.626753:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157184:192.168.203.137@tcp:8:kworker.0 00000800:00000100:2.0:1789766307.626773:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626785:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa5c00 x1876702068157184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626824:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068157184/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626834:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626837:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626852:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157184:192.168.203.137@tcp:8:kworker.0 00000100:00100000:2.0:1789766307.626858:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068157312:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766307.626882:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626894:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa7b80 x1876702068157312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626906:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068157312/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626914:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626917:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626920:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068157312:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766307.626926:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068157440:192.168.203.137@tcp:38:kworker.0 00000800:00000100:2.0:1789766307.626947:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:2.0:1789766307.626959:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766307/real 1789766307] req@ffff936010aa4000 x1876702068157440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789766307.626970:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068157440/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766307.626983:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766307.626986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:2.0:1789766307.626989:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068157440:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766308.010022:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766309.036050:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766310.058651:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766311.085156:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766312.105996:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766312.746973:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.746982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766312.746986:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.746995:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747005:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747011:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766312.747043:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068157568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747076:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747080:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766312.747081:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747085:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747088:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747091:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747100:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766312.747109:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068157696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747117:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747120:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766312.747121:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747125:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747127:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747130:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747133:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766312.747141:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068157824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747148:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747150:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766312.747152:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747155:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747157:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747161:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747165:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766312.747172:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068157952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747179:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747182:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766312.747183:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747186:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747188:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747191:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747195:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766312.747305:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068157568:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766312.747362:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.747401:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dd880 x1876702068157568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.747417:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068157568/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.747426:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.747430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.747433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068157568:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766312.747440:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157696:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766312.747471:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.747487:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dc380 x1876702068157696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.747497:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068157696/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.747504:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.747506:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.747509:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157696:192.168.203.137@tcp:38:kworker.0 00000100:00100000:2.0:1789766312.747620:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068158080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747631:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747635:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766312.747637:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747640:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747644:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747648:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747653:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766312.747666:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068158208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766312.747674:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766312.747676:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766312.747678:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766312.747681:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3780 00000100:00080000:2.0:1789766312.747684:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766312.747686:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766312.747690:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766312.747698:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068158336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766312.747890:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157824:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766312.747914:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.747927:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061df480 x1876702068157824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.747938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068157824/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.747946:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.747949:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.747952:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157824:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766312.747957:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157952:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766312.747979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.747991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dd500 x1876702068157952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.748002:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068157952/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.748010:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.748013:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.748016:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068157952:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766312.748040:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158080:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766312.748059:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.748070:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dc000 x1876702068158080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.748081:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068158080/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.748089:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.748091:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.748094:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158080:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766312.748100:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068158208:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766312.748119:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.748130:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dce00 x1876702068158208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.748142:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068158208/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.748149:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.748152:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.748154:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068158208:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766312.748159:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068158336:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766312.748177:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766312.748187:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766312/real 1789766312] req@ffff9360061dd180 x1876702068158336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766312.748195:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068158336/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766312.748216:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766312.748219:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766312.748222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068158336:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766313.130842:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766314.154123:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766315.181499:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766316.202179:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766317.226125:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766317.865966:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.865977:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766317.865982:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.865994:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866004:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866012:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766317.866048:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df800 x1876702068158464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866082:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866086:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766317.866088:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866092:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866096:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866099:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866108:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766317.866121:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dca80 x1876702068158592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866131:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866134:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766317.866136:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866139:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866142:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866146:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866150:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766317.866159:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068158720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866174:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866177:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766317.866179:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866183:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866186:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866189:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766317.866203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068158848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866211:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866215:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766317.866217:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866220:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866223:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866227:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866231:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766317.866241:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068158976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866250:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866254:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766317.866255:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866259:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866262:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866266:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866270:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766317.866280:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068159104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766317.866288:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766317.866291:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766317.866292:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766317.866296:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3785 00000100:00080000:2.0:1789766317.866298:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766317.866302:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766317.866306:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766317.866314:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068159232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766317.866408:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068158464:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766317.866469:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061df800 x1876702068158464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866523:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df800 x1876702068158464/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866533:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866537:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866540:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068158464:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766317.866548:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158592:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766317.866570:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866581:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061dca80 x1876702068158592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866593:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dca80 x1876702068158592/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866601:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866604:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866608:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158592:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766317.866613:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158720:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766317.866631:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866642:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061dfb80 x1876702068158720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866654:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068158720/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866665:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866668:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158720:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766317.866673:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158848:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766317.866692:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866704:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061de680 x1876702068158848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866716:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068158848/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866725:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866729:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866732:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158848:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766317.866738:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158976:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766317.866755:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866777:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061dd180 x1876702068158976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866790:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068158976/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866822:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866826:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866830:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068158976:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766317.866836:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068159104:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766317.866855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866879:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061dce00 x1876702068159104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866891:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068159104/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866899:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866902:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866905:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068159104:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766317.866911:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068159232:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766317.866929:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766317.866940:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766317/real 1789766317] req@ffff9360061dc000 x1876702068159232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766317.866951:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068159232/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766317.866959:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766317.866962:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766317.866965:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068159232:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766318.250067:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766319.274832:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766320.298495:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766321.322018:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766322.346219:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766322.986939:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766322.986949:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766322.986953:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.986962:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.986972:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.986979:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766322.987013:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068159360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987043:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766322.987047:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766322.987049:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987054:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.987057:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.987061:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766322.987070:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766322.987082:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068159488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987097:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766322.987101:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766322.987102:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987107:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.987110:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.987115:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766322.987119:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766322.987126:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068159360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789766322.987132:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068159616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987143:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766322.987147:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766322.987149:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987153:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.987156:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.987160:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766322.987165:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766322.987177:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068159744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987186:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789766322.987189:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766322.987191:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766322.987195:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987200:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.987204:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.987208:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766322.987212:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766322.987222:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068159872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987230:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789766322.987233:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa5880 x1876702068159360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766322.987236:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766322.987239:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987249:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00080000:3.0:1789766322.987253:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789766322.987254:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068159360/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766322.987262:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789766322.987265:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.987270:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:3.0:1789766322.987270:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766322.987275:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068159360:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766322.987281:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766322.987288:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068160000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766322.987298:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789766322.987302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766322.987305:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766322.987308:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766322.987313:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3790 00000100:00100000:1.0:1789766322.987315:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa4a80 x1876702068159488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766322.987318:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766322.987327:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766322.987328:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068159488/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789766322.987336:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789766322.987337:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.987342:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.987345:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159488:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766322.987354:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068160128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766322.987632:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159616:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766322.987656:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766322.987668:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa6300 x1876702068159616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766322.987678:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068159616/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766322.987685:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.987688:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.987691:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159616:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766322.987696:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159744:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766322.987715:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766322.987924:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa4000 x1876702068159744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766322.987940:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068159744/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766322.987948:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.987951:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.987954:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159744:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766322.987959:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159872:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766322.987979:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766322.987990:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa7b80 x1876702068159872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766322.988000:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068159872/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766322.988008:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.988011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.988013:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068159872:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766322.988018:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160000:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766322.988037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766322.988047:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa5c00 x1876702068160000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766322.988058:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068160000/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766322.988065:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.988068:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.988072:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160000:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766322.988077:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160128:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766322.988095:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766322.988106:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766322/real 1789766322] req@ffff936010aa7800 x1876702068160128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766322.988116:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068160128/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766322.988123:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766322.988125:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766322.988128:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160128:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766323.370002:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766324.394691:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766325.417993:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766326.441990:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766327.466219:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766328.106430:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.106440:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766328.106444:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.106454:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.106464:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.106471:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766328.106505:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068160256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.107138:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.107143:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766328.107146:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.107150:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00100000:1.0:1789766328.107189:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068160256:192.168.203.137@tcp:250:kworker.0 00000100:00080000:2.0:1789766328.109299:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109309:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109322:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766328.109349:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068160384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.109395:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.109399:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766328.109401:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.109406:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.109409:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109412:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109416:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766328.109428:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068160512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.109438:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.109441:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766328.109443:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.109446:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.109449:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109453:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109458:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766328.109468:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068160640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.109477:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.109479:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766328.109481:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.109483:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.109486:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109490:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109493:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766328.109501:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068160768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.109509:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.109512:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766328.109513:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.109527:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.109531:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109534:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109539:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766328.109549:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068160896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766328.109569:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766328.109572:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766328.109574:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766328.109577:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3795 00000100:00080000:2.0:1789766328.109581:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766328.109584:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766328.109588:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766328.109598:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068161024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766328.109638:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766328.109706:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766328.109729:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766328.109750:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766328.109769:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766328.109788:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161024:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766328.490018:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766329.516080:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766330.538197:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789766331.180945:0:2385:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.137:988 00000400:00000100:2.0:1789766331.181088:0:2385:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.137@tcp at host 192.168.203.137 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789766331.181102:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181272:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181281:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181289:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181295:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181302:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000800:00000100:2.0:1789766331.181308:0:2385:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.137@tcp 00000100:00100000:3.0:1789766331.181544:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061df480 x1876702068160384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181575:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068160384/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181586:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181589:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181593:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160384:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766331.181600:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061dc380 x1876702068160512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181609:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068160512/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181619:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181622:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160512:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766331.181627:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061dd880 x1876702068160640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181766:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068160640/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181775:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181778:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181782:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160640:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766331.181788:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061ddc00 x1876702068160768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181826:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068160768/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181834:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181837:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181840:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068160768:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766331.181846:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061df100 x1876702068160896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181858:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068160896/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181865:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181868:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181871:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068160896:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766331.181876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061de300 x1876702068161024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068161024/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181893:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181896:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181899:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161024:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766331.181905:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766328/real 1789766331] req@ffff9360061dd500 x1876702068160256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766331.181915:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068160256/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766331.181922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766331.181925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766331.181928:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068160256:192.168.203.137@tcp:250:kworker.0 00010000:00010000:3.0:1789766331.563968:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766332.586534:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766333.226319:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.226328:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766333.226332:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.226342:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.226352:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.226359:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766333.226393:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068161152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766333.226457:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068161152:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766333.226522:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766333.226551:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.226555:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766333.226557:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.226562:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.226566:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.226570:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.226579:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766333.226593:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068161280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766333.230781:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa5f80 x1876702068161152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.230824:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068161152/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.230835:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.230840:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.230845:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068161152:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766333.230878:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161280:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766333.230935:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.230953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa7480 x1876702068161280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.230965:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068161280/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.230974:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.230977:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.230981:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161280:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766333.231618:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.231625:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766333.231628:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.231635:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.231642:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.231649:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.231672:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766333.231698:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068161408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766333.231711:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.231714:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766333.231715:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.231718:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.231722:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.231725:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.231730:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766333.231739:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068161536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766333.231749:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.231752:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766333.231753:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.231757:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.231760:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.231763:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.231767:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766333.231776:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068161664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766333.231784:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.231787:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766333.231788:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.231791:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.231794:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.231827:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.231833:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766333.231844:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068161792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766333.231852:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766333.231855:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766333.231857:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766333.231860:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3801 00000100:00080000:3.0:1789766333.231864:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766333.231867:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766333.231870:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766333.231878:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068161920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766333.232001:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161408:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766333.232038:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.232054:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa6680 x1876702068161408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.232066:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068161408/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.232074:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.232078:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.232081:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161408:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766333.232087:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161536:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766333.232107:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.232117:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa6a00 x1876702068161536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.232127:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068161536/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.232136:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.232139:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.232141:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161536:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766333.232146:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161664:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766333.232164:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.232175:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa5500 x1876702068161664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.232185:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068161664/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.232191:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.232194:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.232196:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068161664:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766333.232201:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161792:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766333.232219:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.232230:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa5180 x1876702068161792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.232239:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068161792/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.232246:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.232249:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.232252:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161792:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766333.232258:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161920:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766333.232275:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766333.232285:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766333/real 1789766333] req@ffff936010aa4700 x1876702068161920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766333.232294:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068161920/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766333.232300:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766333.232303:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766333.232306:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068161920:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766333.612930:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766334.634593:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766335.659125:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766336.682017:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766337.711332:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766338.346512:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346522:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766338.346526:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346536:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346546:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766338.346553:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766338.346595:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc700 x1876702068162048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346633:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346638:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766338.346640:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346644:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346648:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766338.346652:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766338.346662:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766338.346678:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068162048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789766338.346678:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddf80 x1876702068162176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346694:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346697:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766338.346699:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346703:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346707:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766338.346710:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766338.346715:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766338.346729:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd500 x1876702068162304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346738:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346741:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766338.346744:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346747:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346750:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789766338.346756:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789766338.346757:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766338.346764:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766338.346776:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de300 x1876702068162432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346785:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346788:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766338.346789:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346793:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00100000:1.0:1789766338.346827:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061dc700 x1876702068162048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.346846:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc700 x1876702068162048/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789766338.346850:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766338.346858:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789766338.346859:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00080000:1.0:1789766338.346868:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000008:00100000:2.0:1789766338.346869:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766338.346875:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068162048:192.168.203.137@tcp:250:kworker.0 00000100:00100000:2.0:1789766338.346885:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df100 x1876702068162560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346894:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346898:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766338.346900:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789766338.346905:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162176:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766338.346906:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346912:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766338.346917:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766338.346921:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766338.346930:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061ddc00 x1876702068162688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766338.346938:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766338.346941:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766338.346942:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766338.346946:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3806 00000100:00080000:2.0:1789766338.346948:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766338.346951:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766338.346955:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766338.346968:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd880 x1876702068162816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789766338.347106:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347120:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061ddf80 x1876702068162176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddf80 x1876702068162176/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347141:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347144:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347147:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162176:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766338.347153:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162304:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766338.347171:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347182:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061dd500 x1876702068162304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347193:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd500 x1876702068162304/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347199:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347202:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347205:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162304:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766338.347210:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162432:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766338.347227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347237:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061de300 x1876702068162432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347247:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de300 x1876702068162432/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347254:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347257:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347260:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162432:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766338.347265:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162560:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766338.347282:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347291:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061df100 x1876702068162560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347301:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df100 x1876702068162560/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347307:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347310:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347313:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068162560:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766338.347330:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068162688:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766338.347348:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347358:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061ddc00 x1876702068162688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347367:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061ddc00 x1876702068162688/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347374:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347377:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347380:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061ddc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068162688:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766338.347385:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068162816:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766338.347402:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766338.347411:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766338/real 1789766338] req@ffff9360061dd880 x1876702068162816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766338.347421:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd880 x1876702068162816/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766338.347428:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766338.347430:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766338.347433:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068162816:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766338.730341:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766339.755070:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766340.778031:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766341.802784:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766342.827755:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766343.466671:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.466680:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766343.466684:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.466693:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.466703:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.466711:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766343.466955:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068162944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766343.467023:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068162944:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766343.467082:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766343.467113:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.467117:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766343.467119:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.467124:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.467128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.467132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.467143:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766343.467157:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068163072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766343.469732:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa7100 x1876702068162944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.469759:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068162944/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.469771:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.469775:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.469779:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068162944:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766343.469822:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163072:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766343.469869:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766343.469885:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa4e00 x1876702068163072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.469896:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068163072/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.469904:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.469908:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.469911:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163072:192.168.203.137@tcp:38:kworker.0 00000100:00080000:3.0:1789766343.470073:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.470078:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766343.470081:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.470086:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.470091:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.470097:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.470106:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766343.470345:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068163200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766343.471226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163200:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766343.471263:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:3.0:1789766343.471302:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.471307:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766343.471310:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.471314:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.471318:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.471323:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.471330:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766343.471345:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068163328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766343.471422:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa4380 x1876702068163200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.471437:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068163200/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.471446:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.471450:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.471453:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163200:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766343.471472:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163328:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766343.471495:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766343.471507:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa4e00 x1876702068163328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.471517:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068163328/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.471525:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.471528:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.471531:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163328:192.168.203.137@tcp:8:kworker.0 00000100:00080000:3.0:1789766343.471603:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.471608:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766343.471610:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.471614:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.471618:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.471622:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.471629:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766343.471642:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068163456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766343.471652:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.471655:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766343.471656:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.471660:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.471663:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.471666:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.471671:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766343.471681:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068163584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766343.471689:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766343.471691:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766343.471693:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766343.471696:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3811 00000100:00080000:3.0:1789766343.471700:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766343.471703:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766343.471707:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766343.471717:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068163712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766343.471831:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163456:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766343.471856:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766343.471870:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa7100 x1876702068163456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.471882:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068163456/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.471889:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.471892:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.471895:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163456:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766343.471901:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068163584:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766343.472017:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766343.472030:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa4700 x1876702068163584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.472041:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068163584/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.472049:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.472051:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.472055:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068163584:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766343.472061:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068163712:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766343.472092:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766343.472243:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766343/real 1789766343] req@ffff936010aa5180 x1876702068163712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766343.472255:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068163712/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766343.472262:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766343.472264:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766343.472268:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068163712:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766343.850912:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766344.874016:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766345.898237:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766346.921975:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766347.946006:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766348.586054:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.586064:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766348.586068:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766348.586077:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.586087:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.586094:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766348.586132:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068163840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766348.586163:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.586166:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766348.586168:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766348.586171:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.586174:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.586178:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766348.586187:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766348.586200:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068163968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766348.586208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.586211:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766348.586212:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789766348.586261:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068163840:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789766348.586295:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.586300:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.586304:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766348.586311:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789766348.586321:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766348.586327:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068164096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766348.586340:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.586344:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766348.586346:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766348.586350:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.586354:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.586359:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586364:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef4e00 x1876702068163840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789766348.586367:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766348.586383:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068164224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789766348.586388:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4e00 x1876702068163840/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766348.586403:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766348.586406:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766348.586407:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766348.586411:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766348.586415:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00080000:1.0:1789766348.586419:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00100000:3.0:1789766348.586424:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068163840:192.168.203.137@tcp:250:kworker.0 00000100:00080000:1.0:1789766348.586426:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.586434:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586437:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163968:192.168.203.137@tcp:38:kworker.0 00000008:00100000:1.0:1789766348.586440:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766348.586457:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068164352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789766348.586467:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:1.0:1789766348.586468:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.586473:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766348.586476:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766348.586480:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.586484:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789766348.586486:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef7100 x1876702068163968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766348.586491:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586503:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef7100 x1876702068163968/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.586512:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.586515:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.586518:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068163968:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766348.586555:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164096:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766348.586576:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586587:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef4700 x1876702068164096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766348.586598:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4700 x1876702068164096/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.586606:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.586609:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.586612:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164096:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766348.586617:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164224:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789766348.586634:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586643:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef5180 x1876702068164224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766348.586654:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5180 x1876702068164224/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.586674:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.586677:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.586680:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164224:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766348.586686:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164352:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789766348.586706:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766348.586716:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef5c00 x1876702068164352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766348.586727:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068164352/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.586735:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.586738:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.586740:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164352:192.168.203.137@tcp:8:kworker.0 00000008:00100000:1.0:1789766348.587169:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766348.587187:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068164480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766348.587208:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766348.587212:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789766348.587957:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068164480:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766348.587997:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766348.588014:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef6d80 x1876702068164480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766348.588033:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068164480/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.588042:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.588046:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.588049:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068164480:192.168.203.137@tcp:38:kworker.0 00000100:00080000:1.0:1789766348.588310:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766348.588315:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3816 00000100:00080000:1.0:1789766348.588321:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766348.588327:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766348.588335:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766348.588352:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068164608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789766348.588409:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068164608:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766348.588434:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766348.588446:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766348/real 1789766348] req@ffff936010ef6300 x1876702068164608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766348.588458:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068164608/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766348.588471:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766348.588475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766348.588478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068164608:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766348.973528:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766349.999099:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766351.017989:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766352.043667:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766353.067193:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766353.705925:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.705936:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766353.705940:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.705961:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.705972:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.705979:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766353.706015:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068164736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766353.706046:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.706050:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766353.706052:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706057:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706060:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.706064:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706073:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766353.706085:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068164864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766353.706095:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789766353.706096:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068164736:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766353.706098:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766353.706100:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706103:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706107:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.706110:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706114:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766353.706123:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068164992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766353.706130:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.706133:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766353.706134:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706137:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706140:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.706143:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706147:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789766353.706153:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766353.706154:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068165120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766353.706161:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.706163:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766353.706165:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706167:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706170:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.706173:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706176:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766353.706184:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068165248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766353.706190:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa5500 x1876702068164736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766353.706192:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.706195:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766353.706197:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706200:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706202:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789766353.706205:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068164736/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766353.706206:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706210:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789766353.706215:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789766353.706217:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068165376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766353.706218:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706222:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068164736:192.168.203.137@tcp:250:kworker.0 00000100:00080000:3.0:1789766353.706225:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766353.706228:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766353.706229:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766353.706231:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3821 00000100:00080000:3.0:1789766353.706234:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766353.706237:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766353.706240:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766353.706243:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766353.706247:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068165504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789766353.706261:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706273:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa6a00 x1876702068164864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706284:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068164864/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706292:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706295:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706298:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164864:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766353.706304:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164992:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766353.706322:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706332:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa6680 x1876702068164992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706343:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068164992/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706351:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706354:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706357:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068164992:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766353.706362:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165120:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766353.706382:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706392:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa7480 x1876702068165120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706402:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068165120/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706408:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706411:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706414:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165120:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766353.706419:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165248:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766353.706437:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706448:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa5f80 x1876702068165248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068165248/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706472:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706475:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706478:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165248:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766353.706483:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068165376:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766353.706514:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706526:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa7800 x1876702068165376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706537:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068165376/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706543:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706546:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706549:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068165376:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766353.706571:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068165504:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766353.706590:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766353.706601:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766353/real 1789766353] req@ffff936010aa5c00 x1876702068165504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766353.706611:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068165504/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766353.706623:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766353.706626:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766353.706629:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068165504:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766354.090017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766355.114871:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766356.138203:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766357.162005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766358.186008:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766358.826003:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826013:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766358.826017:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826026:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826036:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826043:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766358.826078:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4000 x1876702068165632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826115:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826119:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766358.826121:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826125:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826128:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826132:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826149:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766358.826161:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4380 x1876702068165760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826182:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826185:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766358.826187:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826191:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826195:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826199:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826204:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766358.826214:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4a80 x1876702068165888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826222:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826225:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766358.826227:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826230:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826233:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826236:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826241:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766358.826250:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5880 x1876702068166016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826258:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826261:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766358.826263:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826266:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826269:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826272:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826277:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766358.826288:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6300 x1876702068166144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826296:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826299:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766358.826300:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826303:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826306:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826310:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826314:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766358.826323:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef6d80 x1876702068166272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766358.826331:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766358.826334:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766358.826336:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766358.826339:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3826 00000100:00080000:1.0:1789766358.826343:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766358.826346:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766358.826351:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766358.826359:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5c00 x1876702068166400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789766358.826425:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068165632:192.168.203.137@tcp:250:kworker.0 00000800:00000100:3.0:1789766358.826493:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826538:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef4000 x1876702068165632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826553:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4000 x1876702068165632/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826561:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826565:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826569:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068165632:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789766358.826575:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165760:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766358.826597:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826608:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef4380 x1876702068165760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826619:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4380 x1876702068165760/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826627:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826630:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826633:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165760:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766358.826642:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165888:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766358.826662:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826672:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef4a80 x1876702068165888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826682:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef4a80 x1876702068165888/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826688:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826691:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826694:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068165888:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766358.826699:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166016:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789766358.826718:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826729:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef5880 x1876702068166016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826739:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5880 x1876702068166016/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826746:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826749:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826752:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166016:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766358.826757:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166144:192.168.203.137@tcp:8:kworker.0 00000800:00000100:3.0:1789766358.826776:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826786:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef6300 x1876702068166144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826818:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6300 x1876702068166144/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826826:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826829:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826832:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166144:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766358.826837:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068166272:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766358.826857:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826867:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef6d80 x1876702068166272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef6d80 x1876702068166272/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826884:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826886:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826889:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068166272:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766358.826893:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068166400:192.168.203.137@tcp:38:kworker.0 00000800:00000100:3.0:1789766358.826910:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:3.0:1789766358.826918:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766358/real 1789766358] req@ffff936010ef5c00 x1876702068166400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789766358.826928:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010ef5c00 x1876702068166400/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789766358.826940:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789766358.826943:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:3.0:1789766358.826946:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010ef5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068166400:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766359.210482:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766360.234314:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766361.257993:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766362.282113:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766363.307257:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766363.945935:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.945945:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766363.945949:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.945958:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947033:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947045:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766363.947084:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7b80 x1876702068166528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947129:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947133:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766363.947136:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947140:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947145:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947149:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947158:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766363.947171:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4000 x1876702068166656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947187:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947191:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766363.947192:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947196:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947200:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947203:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947207:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766363.947218:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6300 x1876702068166784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947226:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947230:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766363.947231:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947235:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947238:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947242:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947246:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766363.947254:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4a80 x1876702068166912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947263:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947266:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766363.947267:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947271:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947274:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947278:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947281:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766363.947290:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5880 x1876702068167040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947299:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947303:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766363.947304:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947308:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947311:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947314:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947319:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766363.947329:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5c00 x1876702068167168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766363.947337:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766363.947340:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766363.947341:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766363.947344:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3831 00000100:00080000:3.0:1789766363.947347:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766363.947351:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766363.947354:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766363.947362:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7800 x1876702068167296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766363.947534:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068166528:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766363.947595:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947635:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa7b80 x1876702068166528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947652:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7b80 x1876702068166528/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.947662:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.947666:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.947669:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068166528:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766363.947676:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166656:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766363.947698:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947712:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa4000 x1876702068166656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947724:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4000 x1876702068166656/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.947733:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.947736:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.947739:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166656:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766363.947746:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166784:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766363.947765:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947776:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa6300 x1876702068166784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947789:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6300 x1876702068166784/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.947822:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.947825:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.947829:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166784:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766363.947835:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166912:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766363.947854:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947865:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa4a80 x1876702068166912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947877:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4a80 x1876702068166912/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.947885:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.947888:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.947892:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068166912:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766363.947898:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167040:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766363.947916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947927:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa5880 x1876702068167040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947938:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5880 x1876702068167040/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.947946:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.947948:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.947952:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167040:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766363.947958:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068167168:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766363.947977:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.947988:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa5c00 x1876702068167168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.947999:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5c00 x1876702068167168/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.948007:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.948011:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.948014:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068167168:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766363.948019:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068167296:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766363.948038:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766363.948049:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766363/real 1789766363] req@ffff936010aa7800 x1876702068167296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766363.948061:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7800 x1876702068167296/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766363.948067:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766363.948070:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766363.948073:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068167296:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766364.329979:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766365.354189:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766366.379309:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766367.402017:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766368.426772:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766369.065872:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.065883:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766369.065887:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.065896:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.065906:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.065913:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766369.065949:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5f80 x1876702068167424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.065978:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.065982:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766369.065983:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.065987:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.065990:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.065994:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066002:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766369.066012:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7480 x1876702068167552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.066021:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.066024:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766369.066026:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.066030:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.066033:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.066036:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066040:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766369.066049:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6680 x1876702068167680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.066057:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.066060:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766369.066062:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.066065:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.066068:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.066071:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066074:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766369.066083:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068167808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.066091:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.066093:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766369.066155:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.066159:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.066162:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.066166:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066169:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766369.066180:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068167936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.066198:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.066201:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766369.066203:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.066210:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.066213:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.066216:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066220:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766369.066238:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068168064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766369.066246:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766369.066249:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766369.066251:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766369.066254:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3836 00000100:00080000:3.0:1789766369.066257:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766369.066260:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766369.066264:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766369.066281:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068168192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766369.066421:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068167424:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766369.066486:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066576:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa5f80 x1876702068167424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066605:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5f80 x1876702068167424/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066616:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066620:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066624:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068167424:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766369.066641:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167552:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766369.066666:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066689:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa7480 x1876702068167552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066702:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7480 x1876702068167552/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066710:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066713:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066717:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167552:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766369.066722:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167680:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766369.066741:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066758:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa6680 x1876702068167680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066770:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6680 x1876702068167680/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066782:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066785:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066788:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167680:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766369.066794:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167808:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766369.066833:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066844:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa6a00 x1876702068167808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066854:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068167808/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066861:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066863:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066867:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167808:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766369.066872:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167936:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766369.066889:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066899:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa5500 x1876702068167936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066911:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068167936/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066918:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066920:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066923:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068167936:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766369.066929:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168064:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766369.066944:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.066953:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa5180 x1876702068168064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.066962:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068168064/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.066968:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.066971:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.066973:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168064:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766369.066978:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168192:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766369.066992:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766369.067001:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766369/real 1789766369] req@ffff936010aa4700 x1876702068168192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766369.067012:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068168192/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766369.067018:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766369.067021:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766369.067023:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168192:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766369.450106:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766370.474081:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766371.498521:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766372.522263:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766373.546151:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789766374.186015:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.186025:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789766374.186030:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.186039:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.186049:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.186055:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:2.0:1789766374.186091:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc380 x1876702068168320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766374.186160:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068168320:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766374.186227:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00080000:2.0:1789766374.186256:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.186260:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766374.186262:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.186265:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.186269:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.186273:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.186282:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766374.186295:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061df480 x1876702068168448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766374.186583:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061dc380 x1876702068168320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.186600:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc380 x1876702068168320/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.186609:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.186613:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.186617:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068168320:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766374.186638:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168448:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766374.186661:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.186673:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061df480 x1876702068168448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.186683:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061df480 x1876702068168448/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.186691:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.186694:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.186697:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061df480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168448:192.168.203.137@tcp:38:kworker.0 00000100:00080000:2.0:1789766374.186913:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.186918:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766374.186919:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.186924:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.186927:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.186931:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.186937:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766374.186951:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dc000 x1876702068168576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766374.186961:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.186964:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789766374.186965:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.186968:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.186972:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.186974:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.186979:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789766374.186988:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dce00 x1876702068168704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766374.186997:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.187000:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1789766374.187001:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.187004:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.187008:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.187011:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.187014:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766374.187023:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dd180 x1876702068168832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766374.187031:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.187034:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789766374.187035:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.187038:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.187041:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.187044:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.187048:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766374.187057:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061de680 x1876702068168960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789766374.187064:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789766374.187067:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789766374.187068:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789766374.187071:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3842 00000100:00080000:2.0:1789766374.187073:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789766374.187076:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:2.0:1789766374.187079:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789766374.187086:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff9360061dfb80 x1876702068169088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766374.187670:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168576:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766374.187705:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.187722:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061dc000 x1876702068168576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.187734:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dc000 x1876702068168576/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.187749:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.187752:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.187756:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dc000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168576:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766374.187762:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168704:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766374.187779:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.187789:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061dce00 x1876702068168704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.187825:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dce00 x1876702068168704/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.187833:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.187836:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.187840:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168704:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766374.187845:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168832:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766374.187864:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.187876:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061dd180 x1876702068168832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.187886:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dd180 x1876702068168832/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.187894:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.187897:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.187901:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068168832:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766374.187906:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168960:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766374.187923:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.187932:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061de680 x1876702068168960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.187943:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061de680 x1876702068168960/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.187951:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.187954:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.187957:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061de680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068168960:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766374.187962:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169088:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766374.187981:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766374.187991:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766374/real 1789766374] req@ffff9360061dfb80 x1876702068169088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766374.188002:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff9360061dfb80 x1876702068169088/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766374.188009:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766374.188012:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766374.188015:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff9360061dfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169088:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766374.570004:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766375.593988:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766376.617985:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766377.644266:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766378.666005:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766379.307453:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307475:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766379.307479:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307490:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307500:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307507:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766379.307545:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadf80 x1876702068169216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307576:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307579:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766379.307581:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307584:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307587:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307592:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307601:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766379.307612:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dadc00 x1876702068169344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307621:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307624:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766379.307626:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307630:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307633:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307636:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307640:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766379.307651:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf800 x1876702068169472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307659:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307661:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766379.307663:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307666:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307669:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307672:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307675:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766379.307684:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daea00 x1876702068169600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307691:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307694:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766379.307696:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307698:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307702:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307705:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307709:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766379.307718:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dace00 x1876702068169728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307727:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307730:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766379.307732:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307735:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307738:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307741:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307745:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766379.307756:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068169216:192.168.203.137@tcp:250:kworker.0 00000100:00100000:0.0:1789766379.307757:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae300 x1876702068169856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766379.307772:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766379.307775:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766379.307777:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766379.307781:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3847 00000100:00080000:0.0:1789766379.307785:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766379.307788:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766379.307793:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766379.307846:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad180 x1876702068169984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789766379.307855:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.307895:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020dadf80 x1876702068169216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.307912:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadf80 x1876702068169216/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.307922:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.307925:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.307929:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068169216:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766379.307936:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169344:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766379.307955:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.307965:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020dadc00 x1876702068169344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.307976:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dadc00 x1876702068169344/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.307983:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.307986:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308002:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dadc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169344:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766379.308009:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169472:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766379.308037:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.308048:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020daf800 x1876702068169472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.308073:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf800 x1876702068169472/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.308085:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.308088:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308091:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169472:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766379.308097:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169600:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766379.308114:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.308124:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020daea00 x1876702068169600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.308133:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daea00 x1876702068169600/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.308141:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.308143:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308146:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169600:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766379.308151:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169728:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766379.308168:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.308177:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020dace00 x1876702068169728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.308188:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dace00 x1876702068169728/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.308195:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.308198:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308201:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dace00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068169728:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766379.308226:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169856:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766379.308248:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.308258:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020dae300 x1876702068169856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.308268:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae300 x1876702068169856/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.308275:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.308277:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308280:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169856:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766379.308285:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169984:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766379.308302:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766379.308325:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766379/real 1789766379] req@ffff936020dad180 x1876702068169984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766379.308354:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad180 x1876702068169984/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766379.308362:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766379.308365:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766379.308368:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068169984:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766379.690506:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766380.714445:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766381.738017:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766382.761994:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766383.786329:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789766384.425861:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.425872:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789766384.425877:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.425887:0:11434:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.425896:0:11434:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.425903:0:11434:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:0.0:1789766384.425942:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac380 x1876702068170112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.425976:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.425980:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766384.425982:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.425986:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.425991:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.425994:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426003:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766384.426016:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dad880 x1876702068170240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.426025:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.426028:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766384.426030:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.426034:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.426038:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.426041:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426046:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766384.426057:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dac000 x1876702068170368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.426065:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.426068:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789766384.426069:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.426072:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.426076:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.426080:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426084:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789766384.426094:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dae680 x1876702068170496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.426102:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.426105:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:0.0:1789766384.426106:0:11434:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.426109:0:11434:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.426113:0:11434:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.426116:0:11434:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426120:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766384.426130:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daed80 x1876702068170624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.426139:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.426142:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789766384.426144:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.426147:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.426150:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.426153:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426158:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766384.426168:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020daf480 x1876702068170752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789766384.426177:0:11434:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789766384.426179:0:11434:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789766384.426181:0:11434:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789766384.426184:0:11434:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3852 00000100:00080000:0.0:1789766384.426187:0:11434:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789766384.426191:0:11434:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:0.0:1789766384.426194:0:11434:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789766384.426203:0:11434:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936020dafb80 x1876702068170880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766384.426348:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068170112:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766384.426409:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426447:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020dac380 x1876702068170112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426464:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac380 x1876702068170112/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426474:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426477:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426481:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068170112:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766384.426488:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170240:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766384.426508:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426518:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020dad880 x1876702068170240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426528:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dad880 x1876702068170240/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426535:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426538:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426541:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dad880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170240:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766384.426546:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170368:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766384.426561:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426570:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020dac000 x1876702068170368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426581:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dac000 x1876702068170368/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426594:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426597:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426600:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dac000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170368:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766384.426606:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170496:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766384.426624:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426634:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020dae680 x1876702068170496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426645:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dae680 x1876702068170496/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426652:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426655:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426658:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dae680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170496:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766384.426663:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170624:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766384.426680:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426691:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020daed80 x1876702068170624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426702:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daed80 x1876702068170624/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426709:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426712:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426716:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068170624:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766384.426721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068170752:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766384.426740:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426750:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020daf480 x1876702068170752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426761:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020daf480 x1876702068170752/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426768:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426771:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426774:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020daf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068170752:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766384.426779:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068170880:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766384.426796:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766384.426914:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766384/real 1789766384] req@ffff936020dafb80 x1876702068170880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766384.426926:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936020dafb80 x1876702068170880/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766384.426933:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766384.426935:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766384.426938:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936020dafb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068170880:192.168.203.137@tcp:38:kworker.0 00010000:00010000:0.0:1789766384.810245:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766385.834150:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766386.859496:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766387.882488:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766388.906701:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789766389.546266:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546276:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789766389.546281:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546290:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546301:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546308:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:3.0:1789766389.546346:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa7100 x1876702068171008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546378:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546382:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766389.546384:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546387:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546391:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546394:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546403:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766389.546414:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4e00 x1876702068171136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546439:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546442:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766389.546443:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546447:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546450:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546453:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546458:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766389.546468:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4380 x1876702068171264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546477:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546480:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789766389.546481:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546485:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546488:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546491:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546495:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789766389.546505:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa4700 x1876702068171392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546513:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546516:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:3.0:1789766389.546518:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546521:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546524:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546528:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546531:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766389.546542:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5180 x1876702068171520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546549:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546552:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789766389.546554:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546556:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546559:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546562:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546566:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766389.546574:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa5500 x1876702068171648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789766389.546582:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789766389.546585:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789766389.546587:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789766389.546589:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3857 00000100:00080000:3.0:1789766389.546593:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789766389.546596:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:3.0:1789766389.546600:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789766389.546608:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010aa6a00 x1876702068171776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789766389.546748:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068171008:192.168.203.137@tcp:250:kworker.0 00000800:00000100:1.0:1789766389.546818:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.546865:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa7100 x1876702068171008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.546879:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa7100 x1876702068171008/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.546887:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.546890:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.546893:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068171008:192.168.203.137@tcp:250:kworker.0 00000100:00100000:1.0:1789766389.546899:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171136:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766389.546916:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.546925:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa4e00 x1876702068171136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.546934:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4e00 x1876702068171136/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.546941:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.546944:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.546946:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171136:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766389.546951:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171264:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766389.546967:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.546977:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa4380 x1876702068171264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.546987:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4380 x1876702068171264/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.547000:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.547003:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.547006:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171264:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766389.547012:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171392:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766389.547030:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.547040:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa4700 x1876702068171392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.547050:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa4700 x1876702068171392/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.547056:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.547059:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.547061:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171392:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766389.547074:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171520:192.168.203.137@tcp:8:kworker.0 00000800:00000100:1.0:1789766389.547089:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.547097:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa5180 x1876702068171520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.547106:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5180 x1876702068171520/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.547111:0:2394:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.547114:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.547125:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068171520:192.168.203.137@tcp:8:kworker.0 00000100:00100000:1.0:1789766389.547129:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068171648:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766389.547143:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.547150:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa5500 x1876702068171648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.547160:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa5500 x1876702068171648/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.547168:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.547171:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.547174:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068171648:192.168.203.137@tcp:38:kworker.0 00000100:00100000:1.0:1789766389.547183:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068171776:192.168.203.137@tcp:38:kworker.0 00000800:00000100:1.0:1789766389.547201:0:2394:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.137@tcp 00000100:00100000:1.0:1789766389.547212:0:2394:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789766389/real 1789766389] req@ffff936010aa6a00 x1876702068171776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789766389.547222:0:2394:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff936010aa6a00 x1876702068171776/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 1 dl 1789766414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789766389.547228:0:2394:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789766389.547231:0:2394:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.137@tcp failed (-110) 00000100:00100000:1.0:1789766389.547233:0:2394:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff936010aa6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068171776:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766389.930284:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766390.955045:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766391.978702:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766393.002109:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789766394.026965:0:6358:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789766394.666012:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 0607efc5-93ac-4424-8fa4-2e162e139892->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666021:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789766394.666025:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf4800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666035:0:8:0:(import.c:510:import_select_connection()) MGC192.168.203.137@tcp: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666045:0:8:0:(import.c:589:import_select_connection()) MGC192.168.203.137@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666052:0:8:0:(import.c:624:import_select_connection()) MGC192.168.203.137@tcp: import ffff936010bf4800 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000100:00100000:1.0:1789766394.666148:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5180 x1876702068171904/t0(0) o250->MGC192.168.203.137@tcp@192.168.203.137@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666177:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666180:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766394.666182:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003e69000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666186:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666190:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666194:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936020592000: import ffff936003e69000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666211:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766394.666222:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4700 x1876702068172032/t0(0) o38->lustre-MDT0000-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666233:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766394.666235:0:8:0:(import.c:65:import_set_state_nolock()) ffff936003c9c000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666238:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666241:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666245:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936020592000: import ffff936003c9c000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666249:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766394.666256:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7100 x1876702068172160/t0(0) o38->lustre-MDT0001-mdc-ffff936020592000@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666270:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666273:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789766394.666274:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b7000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666278:0:8:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666281:0:8:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666283:0:8:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff936020592000: import ffff93601a4b7000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666287:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789766394.666294:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef4e00 x1876702068172288/t0(0) o8->lustre-OST0000-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666301:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) fa6fc127-0d58-47ca-82d1-2c8ad7b086c1->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666304:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1789766394.666306:0:8:0:(import.c:65:import_set_state_nolock()) ffff93601a4b0000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666308:0:8:0:(import.c:510:import_select_connection()) lustre-OST0001-osc-ffff936020592000: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666311:0:8:0:(import.c:589:import_select_connection()) lustre-OST0001-osc-ffff936020592000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666314:0:8:0:(import.c:624:import_select_connection()) lustre-OST0001-osc-ffff936020592000: import ffff93601a4b0000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666318:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766394.666325:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7b80 x1876702068172416/t0(0) o8->lustre-OST0001-osc-ffff936020592000@192.168.203.137@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666332:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666335:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789766394.666336:0:8:0:(import.c:65:import_set_state_nolock()) ffff936010bf6000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666339:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666342:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666345:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff936008b5a800: import ffff936010bf6000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666348:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766394.666356:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef7800 x1876702068172544/t0(0) o38->lustre-MDT0000-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789766394.666363:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) 2a924629-ba65-41c7-b7b8-60afcac2e2ea->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789766394.666365:0:8:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789766394.666367:0:8:0:(import.c:65:import_set_state_nolock()) ffff936006b46000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789766394.666370:0:8:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: connect to NID 192.168.203.137@tcp last attempt 3862 00000100:00080000:1.0:1789766394.666372:0:8:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789766394.666375:0:8:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff936008b5a800: import ffff936006b46000 using connection 192.168.203.137@tcp/192.168.203.137@tcp 00000008:00100000:1.0:1789766394.666378:0:8:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683242 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789766394.666386:0:8:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff936010ef5500 x1876702068172672/t0(0) o38->lustre-MDT0001-mdc-ffff936008b5a800@192.168.203.137@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789766394.666465:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:0607efc5-93ac-4424-8fa4-2e162e139892:2394:1876702068171904:192.168.203.137@tcp:250:kworker.0 00000100:00100000:3.0:1789766394.666618:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068172032:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766394.666648:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068172160:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766394.666669:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068172288:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766394.666687:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:fa6fc127-0d58-47ca-82d1-2c8ad7b086c1:2394:1876702068172416:192.168.203.137@tcp:8:kworker.0 00000100:00100000:3.0:1789766394.666704:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068172544:192.168.203.137@tcp:38:kworker.0 00000100:00100000:3.0:1789766394.666721:0:2394:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff936010ef5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:2a924629-ba65-41c7-b7b8-60afcac2e2ea:2394:1876702068172672:192.168.203.137@tcp:38:kworker.0 00010000:00010000:3.0:1789766395.050016:0:6357:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789766396.074344:0:6823:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 80021 lines, 80021 kept, 0 dropped, 0 bad.